268 lines
9.3 KiB
JavaScript
268 lines
9.3 KiB
JavaScript
import { promises as fs, existsSync } from "node:fs";
|
|
import PQueue from "p-queue";
|
|
import xxhash from "xxhash-wasm";
|
|
import { AstroUserError } from "../core/errors/errors.js";
|
|
import {
|
|
ASSET_IMPORTS_FILE,
|
|
CONTENT_LAYER_TYPE,
|
|
DATA_STORE_FILE,
|
|
MODULES_IMPORTS_FILE
|
|
} from "./consts.js";
|
|
import {
|
|
getEntryConfigByExtMap,
|
|
getEntryDataAndImages,
|
|
globalContentConfigObserver
|
|
} from "./utils.js";
|
|
class ContentLayer {
|
|
#logger;
|
|
#store;
|
|
#settings;
|
|
#watcher;
|
|
#lastConfigDigest;
|
|
#unsubscribe;
|
|
#generateDigest;
|
|
#queue;
|
|
constructor({ settings, logger, store, watcher }) {
|
|
watcher?.setMaxListeners(50);
|
|
this.#logger = logger;
|
|
this.#store = store;
|
|
this.#settings = settings;
|
|
this.#watcher = watcher;
|
|
this.#queue = new PQueue({ concurrency: 1 });
|
|
}
|
|
/**
|
|
* Whether the content layer is currently loading content
|
|
*/
|
|
get loading() {
|
|
return this.#queue.size > 0 || this.#queue.pending > 0;
|
|
}
|
|
/**
|
|
* Watch for changes to the content config and trigger a sync when it changes.
|
|
*/
|
|
watchContentConfig() {
|
|
this.#unsubscribe?.();
|
|
this.#unsubscribe = globalContentConfigObserver.subscribe(async (ctx) => {
|
|
if (ctx.status === "loaded" && ctx.config.digest !== this.#lastConfigDigest) {
|
|
this.sync();
|
|
}
|
|
});
|
|
}
|
|
unwatchContentConfig() {
|
|
this.#unsubscribe?.();
|
|
}
|
|
async #getGenerateDigest() {
|
|
if (this.#generateDigest) {
|
|
return this.#generateDigest;
|
|
}
|
|
const { h64ToString } = await xxhash();
|
|
this.#generateDigest = (data) => {
|
|
const dataString = typeof data === "string" ? data : JSON.stringify(data);
|
|
return h64ToString(dataString);
|
|
};
|
|
return this.#generateDigest;
|
|
}
|
|
async #getLoaderContext({
|
|
collectionName,
|
|
loaderName = "content",
|
|
parseData,
|
|
refreshContextData
|
|
}) {
|
|
return {
|
|
collection: collectionName,
|
|
store: this.#store.scopedStore(collectionName),
|
|
meta: this.#store.metaStore(collectionName),
|
|
logger: this.#logger.forkIntegrationLogger(loaderName),
|
|
config: this.#settings.config,
|
|
parseData,
|
|
generateDigest: await this.#getGenerateDigest(),
|
|
watcher: this.#watcher,
|
|
refreshContextData,
|
|
entryTypes: getEntryConfigByExtMap([
|
|
...this.#settings.contentEntryTypes,
|
|
...this.#settings.dataEntryTypes
|
|
])
|
|
};
|
|
}
|
|
/**
|
|
* Enqueues a sync job that runs the `load()` method of each collection's loader, which will load the data and save it in the data store.
|
|
* The loader itself is responsible for deciding whether this will clear and reload the full collection, or
|
|
* perform an incremental update. After the data is loaded, the data store is written to disk. Jobs are queued,
|
|
* so that only one sync can run at a time. The function returns a promise that resolves when this sync job is complete.
|
|
*/
|
|
sync(options = {}) {
|
|
return this.#queue.add(() => this.#doSync(options));
|
|
}
|
|
async #doSync(options) {
|
|
const contentConfig = globalContentConfigObserver.get();
|
|
const logger = this.#logger.forkIntegrationLogger("content");
|
|
if (contentConfig?.status !== "loaded") {
|
|
logger.debug("Content config not loaded, skipping sync");
|
|
return;
|
|
}
|
|
if (!this.#settings.config.experimental.contentLayer) {
|
|
const contentLayerCollections = Object.entries(contentConfig.config.collections).filter(
|
|
([_, collection]) => collection.type === CONTENT_LAYER_TYPE
|
|
);
|
|
if (contentLayerCollections.length > 0) {
|
|
throw new AstroUserError(
|
|
`The following collections have a loader defined, but the content layer is not enabled: ${contentLayerCollections.map(([title]) => title).join(", ")}.`,
|
|
"To enable the Content Layer API, set `experimental: { contentLayer: true }` in your Astro config file."
|
|
);
|
|
}
|
|
return;
|
|
}
|
|
logger.info("Syncing content");
|
|
const { digest: currentConfigDigest } = contentConfig.config;
|
|
this.#lastConfigDigest = currentConfigDigest;
|
|
let shouldClear = false;
|
|
const previousConfigDigest = await this.#store.metaStore().get("config-digest");
|
|
const previousAstroVersion = await this.#store.metaStore().get("astro-version");
|
|
if (currentConfigDigest && previousConfigDigest !== currentConfigDigest) {
|
|
logger.info("Content config changed");
|
|
shouldClear = true;
|
|
}
|
|
if (previousAstroVersion !== "4.16.18") {
|
|
logger.info("Astro version changed");
|
|
shouldClear = true;
|
|
}
|
|
if (shouldClear) {
|
|
logger.info("Clearing content store");
|
|
this.#store.clearAll();
|
|
}
|
|
if ("4.16.18") {
|
|
await this.#store.metaStore().set("astro-version", "4.16.18");
|
|
}
|
|
if (currentConfigDigest) {
|
|
await this.#store.metaStore().set("config-digest", currentConfigDigest);
|
|
}
|
|
await Promise.all(
|
|
Object.entries(contentConfig.config.collections).map(async ([name, collection]) => {
|
|
if (collection.type !== CONTENT_LAYER_TYPE) {
|
|
return;
|
|
}
|
|
let { schema } = collection;
|
|
if (!schema && typeof collection.loader === "object") {
|
|
schema = collection.loader.schema;
|
|
if (typeof schema === "function") {
|
|
schema = await schema();
|
|
}
|
|
}
|
|
if (options?.loaders && (typeof collection.loader !== "object" || !options.loaders.includes(collection.loader.name))) {
|
|
return;
|
|
}
|
|
const collectionWithResolvedSchema = { ...collection, schema };
|
|
const parseData = async ({ id, data, filePath = "" }) => {
|
|
const { data: parsedData } = await getEntryDataAndImages(
|
|
{
|
|
id,
|
|
collection: name,
|
|
unvalidatedData: data,
|
|
_internal: {
|
|
rawData: void 0,
|
|
filePath
|
|
}
|
|
},
|
|
collectionWithResolvedSchema,
|
|
false
|
|
);
|
|
return parsedData;
|
|
};
|
|
const context = await this.#getLoaderContext({
|
|
collectionName: name,
|
|
parseData,
|
|
loaderName: collection.loader.name,
|
|
refreshContextData: options?.context
|
|
});
|
|
if (typeof collection.loader === "function") {
|
|
return simpleLoader(collection.loader, context);
|
|
}
|
|
if (!collection.loader.load) {
|
|
throw new Error(`Collection loader for ${name} does not have a load method`);
|
|
}
|
|
return collection.loader.load(context);
|
|
})
|
|
);
|
|
if (!existsSync(this.#settings.config.cacheDir)) {
|
|
await fs.mkdir(this.#settings.config.cacheDir, { recursive: true });
|
|
}
|
|
const cacheFile = getDataStoreFile(this.#settings);
|
|
await this.#store.writeToDisk(cacheFile);
|
|
if (!existsSync(this.#settings.dotAstroDir)) {
|
|
await fs.mkdir(this.#settings.dotAstroDir, { recursive: true });
|
|
}
|
|
const assetImportsFile = new URL(ASSET_IMPORTS_FILE, this.#settings.dotAstroDir);
|
|
await this.#store.writeAssetImports(assetImportsFile);
|
|
const modulesImportsFile = new URL(MODULES_IMPORTS_FILE, this.#settings.dotAstroDir);
|
|
await this.#store.writeModuleImports(modulesImportsFile);
|
|
logger.info("Synced content");
|
|
if (this.#settings.config.experimental.contentIntellisense) {
|
|
await this.regenerateCollectionFileManifest();
|
|
}
|
|
}
|
|
async regenerateCollectionFileManifest() {
|
|
const collectionsManifest = new URL("collections/collections.json", this.#settings.dotAstroDir);
|
|
this.#logger.debug("content", "Regenerating collection file manifest");
|
|
if (existsSync(collectionsManifest)) {
|
|
try {
|
|
const collections = await fs.readFile(collectionsManifest, "utf-8");
|
|
const collectionsJson = JSON.parse(collections);
|
|
collectionsJson.entries ??= {};
|
|
for (const { hasSchema, name } of collectionsJson.collections) {
|
|
if (!hasSchema) {
|
|
continue;
|
|
}
|
|
const entries = this.#store.values(name);
|
|
if (!entries?.[0]?.filePath) {
|
|
continue;
|
|
}
|
|
for (const { filePath } of entries) {
|
|
if (!filePath) {
|
|
continue;
|
|
}
|
|
const key = new URL(filePath, this.#settings.config.root).href.toLowerCase();
|
|
collectionsJson.entries[key] = name;
|
|
}
|
|
}
|
|
await fs.writeFile(collectionsManifest, JSON.stringify(collectionsJson, null, 2));
|
|
} catch {
|
|
this.#logger.error("content", "Failed to regenerate collection file manifest");
|
|
}
|
|
}
|
|
this.#logger.debug("content", "Regenerated collection file manifest");
|
|
}
|
|
}
|
|
async function simpleLoader(handler, context) {
|
|
const data = await handler();
|
|
context.store.clear();
|
|
for (const raw of data) {
|
|
const item = await context.parseData({ id: raw.id, data: raw });
|
|
context.store.set({ id: raw.id, data: item });
|
|
}
|
|
}
|
|
function getDataStoreFile(settings, isDev) {
|
|
isDev ??= process?.env.NODE_ENV === "development";
|
|
return new URL(DATA_STORE_FILE, isDev ? settings.dotAstroDir : settings.config.cacheDir);
|
|
}
|
|
function contentLayerSingleton() {
|
|
let instance = null;
|
|
return {
|
|
init: (options) => {
|
|
instance?.unwatchContentConfig();
|
|
instance = new ContentLayer(options);
|
|
return instance;
|
|
},
|
|
get: () => instance,
|
|
dispose: () => {
|
|
instance?.unwatchContentConfig();
|
|
instance = null;
|
|
}
|
|
};
|
|
}
|
|
const globalContentLayer = contentLayerSingleton();
|
|
export {
|
|
ContentLayer,
|
|
getDataStoreFile,
|
|
globalContentLayer,
|
|
simpleLoader
|
|
};
|