full site update
This commit is contained in:
180
node_modules/astro/dist/content/content-layer.js
generated
vendored
180
node_modules/astro/dist/content/content-layer.js
generated
vendored
@@ -1,9 +1,11 @@
|
||||
import { promises as fs, existsSync } from "node:fs";
|
||||
import { existsSync, promises as fs } from "node:fs";
|
||||
import { createMarkdownProcessor } from "@astrojs/markdown-remark";
|
||||
import PQueue from "p-queue";
|
||||
import xxhash from "xxhash-wasm";
|
||||
import { AstroUserError } from "../core/errors/errors.js";
|
||||
import { AstroError, AstroErrorData } from "../core/errors/index.js";
|
||||
import {
|
||||
ASSET_IMPORTS_FILE,
|
||||
COLLECTIONS_MANIFEST_FILE,
|
||||
CONTENT_LAYER_TYPE,
|
||||
DATA_STORE_FILE,
|
||||
MODULES_IMPORTS_FILE
|
||||
@@ -11,8 +13,11 @@ import {
|
||||
import {
|
||||
getEntryConfigByExtMap,
|
||||
getEntryDataAndImages,
|
||||
globalContentConfigObserver
|
||||
globalContentConfigObserver,
|
||||
loaderReturnSchema,
|
||||
safeStringify
|
||||
} from "./utils.js";
|
||||
import { createWatcherWrapper } from "./watcher.js";
|
||||
class ContentLayer {
|
||||
#logger;
|
||||
#store;
|
||||
@@ -20,6 +25,7 @@ class ContentLayer {
|
||||
#watcher;
|
||||
#lastConfigDigest;
|
||||
#unsubscribe;
|
||||
#markdownProcessor;
|
||||
#generateDigest;
|
||||
#queue;
|
||||
constructor({ settings, logger, store, watcher }) {
|
||||
@@ -27,7 +33,9 @@ class ContentLayer {
|
||||
this.#logger = logger;
|
||||
this.#store = store;
|
||||
this.#settings = settings;
|
||||
this.#watcher = watcher;
|
||||
if (watcher) {
|
||||
this.#watcher = createWatcherWrapper(watcher);
|
||||
}
|
||||
this.#queue = new PQueue({ concurrency: 1 });
|
||||
}
|
||||
/**
|
||||
@@ -50,6 +58,11 @@ class ContentLayer {
|
||||
unwatchContentConfig() {
|
||||
this.#unsubscribe?.();
|
||||
}
|
||||
dispose() {
|
||||
this.#queue.clear();
|
||||
this.#unsubscribe?.();
|
||||
this.#watcher?.removeAllTrackedListeners();
|
||||
}
|
||||
async #getGenerateDigest() {
|
||||
if (this.#generateDigest) {
|
||||
return this.#generateDigest;
|
||||
@@ -74,6 +87,7 @@ class ContentLayer {
|
||||
logger: this.#logger.forkIntegrationLogger(loaderName),
|
||||
config: this.#settings.config,
|
||||
parseData,
|
||||
renderMarkdown: this.#processMarkdown.bind(this),
|
||||
generateDigest: await this.#getGenerateDigest(),
|
||||
watcher: this.#watcher,
|
||||
refreshContextData,
|
||||
@@ -83,6 +97,14 @@ class ContentLayer {
|
||||
])
|
||||
};
|
||||
}
|
||||
async #processMarkdown(content) {
|
||||
this.#markdownProcessor ??= await createMarkdownProcessor(this.#settings.config.markdown);
|
||||
const { code, metadata } = await this.#markdownProcessor.render(content);
|
||||
return {
|
||||
html: code,
|
||||
metadata
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Enqueues a sync job that runs the `load()` method of each collection's loader, which will load the data and save it in the data store.
|
||||
* The loader itself is responsible for deciding whether this will clear and reload the full collection, or
|
||||
@@ -93,35 +115,56 @@ class ContentLayer {
|
||||
return this.#queue.add(() => this.#doSync(options));
|
||||
}
|
||||
async #doSync(options) {
|
||||
const contentConfig = globalContentConfigObserver.get();
|
||||
let contentConfig = globalContentConfigObserver.get();
|
||||
const logger = this.#logger.forkIntegrationLogger("content");
|
||||
if (contentConfig?.status !== "loaded") {
|
||||
logger.debug("Content config not loaded, skipping sync");
|
||||
if (contentConfig?.status === "loading") {
|
||||
contentConfig = await Promise.race([
|
||||
new Promise((resolve) => {
|
||||
const unsub = globalContentConfigObserver.subscribe((ctx) => {
|
||||
unsub();
|
||||
resolve(ctx);
|
||||
});
|
||||
}),
|
||||
new Promise(
|
||||
(resolve) => setTimeout(
|
||||
() => resolve({ status: "error", error: new Error("Content config loading timed out") }),
|
||||
5e3
|
||||
)
|
||||
)
|
||||
]);
|
||||
}
|
||||
if (contentConfig?.status === "error") {
|
||||
logger.error(`Error loading content config. Skipping sync.
|
||||
${contentConfig.error.message}`);
|
||||
return;
|
||||
}
|
||||
if (!this.#settings.config.experimental.contentLayer) {
|
||||
const contentLayerCollections = Object.entries(contentConfig.config.collections).filter(
|
||||
([_, collection]) => collection.type === CONTENT_LAYER_TYPE
|
||||
);
|
||||
if (contentLayerCollections.length > 0) {
|
||||
throw new AstroUserError(
|
||||
`The following collections have a loader defined, but the content layer is not enabled: ${contentLayerCollections.map(([title]) => title).join(", ")}.`,
|
||||
"To enable the Content Layer API, set `experimental: { contentLayer: true }` in your Astro config file."
|
||||
);
|
||||
}
|
||||
if (contentConfig?.status !== "loaded") {
|
||||
logger.error(`Content config not loaded, skipping sync. Status was ${contentConfig?.status}`);
|
||||
return;
|
||||
}
|
||||
logger.info("Syncing content");
|
||||
const {
|
||||
vite: _vite,
|
||||
integrations: _integrations,
|
||||
adapter: _adapter,
|
||||
...hashableConfig
|
||||
} = this.#settings.config;
|
||||
const astroConfigDigest = safeStringify(hashableConfig);
|
||||
const { digest: currentConfigDigest } = contentConfig.config;
|
||||
this.#lastConfigDigest = currentConfigDigest;
|
||||
let shouldClear = false;
|
||||
const previousConfigDigest = await this.#store.metaStore().get("config-digest");
|
||||
const previousConfigDigest = await this.#store.metaStore().get("content-config-digest");
|
||||
const previousAstroConfigDigest = await this.#store.metaStore().get("astro-config-digest");
|
||||
const previousAstroVersion = await this.#store.metaStore().get("astro-version");
|
||||
if (currentConfigDigest && previousConfigDigest !== currentConfigDigest) {
|
||||
if (previousAstroConfigDigest && previousAstroConfigDigest !== astroConfigDigest) {
|
||||
logger.info("Astro config changed");
|
||||
shouldClear = true;
|
||||
}
|
||||
if (previousConfigDigest && previousConfigDigest !== currentConfigDigest) {
|
||||
logger.info("Content config changed");
|
||||
shouldClear = true;
|
||||
}
|
||||
if (previousAstroVersion !== "4.16.18") {
|
||||
if (previousAstroVersion && previousAstroVersion !== "5.12.3") {
|
||||
logger.info("Astro version changed");
|
||||
shouldClear = true;
|
||||
}
|
||||
@@ -129,11 +172,17 @@ class ContentLayer {
|
||||
logger.info("Clearing content store");
|
||||
this.#store.clearAll();
|
||||
}
|
||||
if ("4.16.18") {
|
||||
await this.#store.metaStore().set("astro-version", "4.16.18");
|
||||
if ("5.12.3") {
|
||||
await this.#store.metaStore().set("astro-version", "5.12.3");
|
||||
}
|
||||
if (currentConfigDigest) {
|
||||
await this.#store.metaStore().set("config-digest", currentConfigDigest);
|
||||
await this.#store.metaStore().set("content-config-digest", currentConfigDigest);
|
||||
}
|
||||
if (astroConfigDigest) {
|
||||
await this.#store.metaStore().set("astro-config-digest", astroConfigDigest);
|
||||
}
|
||||
if (!options?.loaders?.length) {
|
||||
this.#watcher?.removeAllTrackedListeners();
|
||||
}
|
||||
await Promise.all(
|
||||
Object.entries(contentConfig.config.collections).map(async ([name, collection]) => {
|
||||
@@ -163,7 +212,9 @@ class ContentLayer {
|
||||
}
|
||||
},
|
||||
collectionWithResolvedSchema,
|
||||
false
|
||||
false,
|
||||
// FUTURE: Remove in this in v6
|
||||
id.endsWith(".svg")
|
||||
);
|
||||
return parsedData;
|
||||
};
|
||||
@@ -182,25 +233,20 @@ class ContentLayer {
|
||||
return collection.loader.load(context);
|
||||
})
|
||||
);
|
||||
if (!existsSync(this.#settings.config.cacheDir)) {
|
||||
await fs.mkdir(this.#settings.config.cacheDir, { recursive: true });
|
||||
}
|
||||
const cacheFile = getDataStoreFile(this.#settings);
|
||||
await this.#store.writeToDisk(cacheFile);
|
||||
if (!existsSync(this.#settings.dotAstroDir)) {
|
||||
await fs.mkdir(this.#settings.dotAstroDir, { recursive: true });
|
||||
}
|
||||
await fs.mkdir(this.#settings.config.cacheDir, { recursive: true });
|
||||
await fs.mkdir(this.#settings.dotAstroDir, { recursive: true });
|
||||
const assetImportsFile = new URL(ASSET_IMPORTS_FILE, this.#settings.dotAstroDir);
|
||||
await this.#store.writeAssetImports(assetImportsFile);
|
||||
const modulesImportsFile = new URL(MODULES_IMPORTS_FILE, this.#settings.dotAstroDir);
|
||||
await this.#store.writeModuleImports(modulesImportsFile);
|
||||
await this.#store.waitUntilSaveComplete();
|
||||
logger.info("Synced content");
|
||||
if (this.#settings.config.experimental.contentIntellisense) {
|
||||
await this.regenerateCollectionFileManifest();
|
||||
}
|
||||
}
|
||||
async regenerateCollectionFileManifest() {
|
||||
const collectionsManifest = new URL("collections/collections.json", this.#settings.dotAstroDir);
|
||||
const collectionsManifest = new URL(COLLECTIONS_MANIFEST_FILE, this.#settings.dotAstroDir);
|
||||
this.#logger.debug("content", "Regenerating collection file manifest");
|
||||
if (existsSync(collectionsManifest)) {
|
||||
try {
|
||||
@@ -232,36 +278,82 @@ class ContentLayer {
|
||||
}
|
||||
}
|
||||
async function simpleLoader(handler, context) {
|
||||
const data = await handler();
|
||||
context.store.clear();
|
||||
for (const raw of data) {
|
||||
const item = await context.parseData({ id: raw.id, data: raw });
|
||||
context.store.set({ id: raw.id, data: item });
|
||||
const unsafeData = await handler();
|
||||
const parsedData = loaderReturnSchema.safeParse(unsafeData);
|
||||
if (!parsedData.success) {
|
||||
const issue = parsedData.error.issues[0];
|
||||
const parseIssue = Array.isArray(unsafeData) ? issue.unionErrors[0] : issue.unionErrors[1];
|
||||
const error = parseIssue.errors[0];
|
||||
const firstPathItem = error.path[0];
|
||||
const entry = Array.isArray(unsafeData) ? unsafeData[firstPathItem] : unsafeData[firstPathItem];
|
||||
throw new AstroError({
|
||||
...AstroErrorData.ContentLoaderReturnsInvalidId,
|
||||
message: AstroErrorData.ContentLoaderReturnsInvalidId.message(context.collection, entry)
|
||||
});
|
||||
}
|
||||
const data = parsedData.data;
|
||||
context.store.clear();
|
||||
if (Array.isArray(data)) {
|
||||
for (const raw of data) {
|
||||
if (!raw.id) {
|
||||
throw new AstroError({
|
||||
...AstroErrorData.ContentLoaderInvalidDataError,
|
||||
message: AstroErrorData.ContentLoaderInvalidDataError.message(
|
||||
context.collection,
|
||||
`Entry missing ID:
|
||||
${JSON.stringify({ ...raw, id: void 0 }, null, 2)}`
|
||||
)
|
||||
});
|
||||
}
|
||||
const item = await context.parseData({ id: raw.id, data: raw });
|
||||
context.store.set({ id: raw.id, data: item });
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (typeof data === "object") {
|
||||
for (const [id, raw] of Object.entries(data)) {
|
||||
if (raw.id && raw.id !== id) {
|
||||
throw new AstroError({
|
||||
...AstroErrorData.ContentLoaderInvalidDataError,
|
||||
message: AstroErrorData.ContentLoaderInvalidDataError.message(
|
||||
context.collection,
|
||||
`Object key ${JSON.stringify(id)} does not match ID ${JSON.stringify(raw.id)}`
|
||||
)
|
||||
});
|
||||
}
|
||||
const item = await context.parseData({ id, data: raw });
|
||||
context.store.set({ id, data: item });
|
||||
}
|
||||
return;
|
||||
}
|
||||
throw new AstroError({
|
||||
...AstroErrorData.ExpectedImageOptions,
|
||||
message: AstroErrorData.ContentLoaderInvalidDataError.message(
|
||||
context.collection,
|
||||
`Invalid data type: ${typeof data}`
|
||||
)
|
||||
});
|
||||
}
|
||||
function getDataStoreFile(settings, isDev) {
|
||||
isDev ??= process?.env.NODE_ENV === "development";
|
||||
return new URL(DATA_STORE_FILE, isDev ? settings.dotAstroDir : settings.config.cacheDir);
|
||||
}
|
||||
function contentLayerSingleton() {
|
||||
let instance = null;
|
||||
return {
|
||||
init: (options) => {
|
||||
instance?.unwatchContentConfig();
|
||||
instance?.dispose();
|
||||
instance = new ContentLayer(options);
|
||||
return instance;
|
||||
},
|
||||
get: () => instance,
|
||||
dispose: () => {
|
||||
instance?.unwatchContentConfig();
|
||||
instance?.dispose();
|
||||
instance = null;
|
||||
}
|
||||
};
|
||||
}
|
||||
const globalContentLayer = contentLayerSingleton();
|
||||
export {
|
||||
ContentLayer,
|
||||
getDataStoreFile,
|
||||
globalContentLayer,
|
||||
simpleLoader
|
||||
globalContentLayer
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user