fix: keep data store outside of node_modules during dev (#11902)

* fix: don't keep data store in node_modules during dev

* Lint

* Fix test

* Wait for data store

* Use helper for data store file

* Fix data store file helper

* Lint

* Handle case where Vite already knows about save

.changeset/forty-spies-train.md

@@ -0,0 +1,5 @@
+---
+'astro': patch
+---
+
+Fixes case where content layer did not update during clean dev builds on Linux and Windows

packages/astro/src/content/content-layer.ts

@@ -223,7 +223,7 @@ export class ContentLayer {
 		if (!existsSync(this.#settings.config.cacheDir)) {
 			await fs.mkdir(this.#settings.config.cacheDir, { recursive: true });
 		}
-		const cacheFile = new URL(DATA_STORE_FILE, this.#settings.config.cacheDir);
+		const cacheFile = getDataStoreFile(this.#settings);
 		await this.#store.writeToDisk(cacheFile);
 		if (!existsSync(this.#settings.dotAstroDir)) {
 			await fs.mkdir(this.#settings.dotAstroDir, { recursive: true });
@@ -283,6 +283,15 @@ export async function simpleLoader<TData extends { id: string }>(
 		context.store.set({ id: raw.id, data: item });
 	}
 }
+/**
+ * Get the path to the data store file.
+ * During development, this is in the `.astro` directory so that the Vite watcher can see it.
+ * In production, it's in the cache directory so that it's preserved between builds.
+ */
+export function getDataStoreFile(settings: AstroSettings, isDev?: boolean) {
+	isDev ??= process?.env.NODE_ENV === 'development';
+	return new URL(DATA_STORE_FILE, isDev ? settings.dotAstroDir : settings.config.cacheDir);
+}
 
 function contentLayerSingleton() {
 	let instance: ContentLayer | null = null;

packages/astro/src/content/data-store.ts

@@ -91,6 +91,9 @@ export class DataStore {
 		try {
 			// @ts-expect-error - this is a virtual module
 			const data = await import('astro:data-layer-content');
+			if (data.default instanceof Map) {
+				return DataStore.fromMap(data.default);
+			}
 			const map = devalue.unflatten(data.default);
 			return DataStore.fromMap(map);
 		} catch {}

packages/astro/src/content/vite-plugin-content-virtual-mod.ts

@@ -20,7 +20,6 @@ import {
 	CONTENT_FLAG,
 	CONTENT_RENDER_FLAG,
 	DATA_FLAG,
-	DATA_STORE_FILE,
 	DATA_STORE_VIRTUAL_ID,
 	MODULES_IMPORTS_FILE,
 	MODULES_MJS_ID,
@@ -29,6 +28,7 @@ import {
 	RESOLVED_VIRTUAL_MODULE_ID,
 	VIRTUAL_MODULE_ID,
 } from './consts.js';
+import { getDataStoreFile } from './content-layer.js';
 import {
 	type ContentLookupMap,
 	getContentEntryIdAndSlug,
@@ -54,12 +54,13 @@ export function astroContentVirtualModPlugin({
 }: AstroContentVirtualModPluginParams): Plugin {
 	let IS_DEV = false;
 	const IS_SERVER = isServerLikeOutput(settings.config);
-	const dataStoreFile = new URL(DATA_STORE_FILE, settings.config.cacheDir);
+	let dataStoreFile: URL;
 	return {
 		name: 'astro-content-virtual-mod-plugin',
 		enforce: 'pre',
 		configResolved(config) {
 			IS_DEV = config.mode === 'development';
+			dataStoreFile = getDataStoreFile(settings, IS_DEV);
 		},
 		async resolveId(id) {
 			if (id === VIRTUAL_MODULE_ID) {
@@ -180,25 +181,31 @@ export function astroContentVirtualModPlugin({
 
 		configureServer(server) {
 			const dataStorePath = fileURLToPath(dataStoreFile);
-			// Watch for changes to the data store file
-			if (Array.isArray(server.watcher.options.ignored)) {
-				// The data store file is in node_modules, so is ignored by default,
-				// so we need to un-ignore it.
-				server.watcher.options.ignored.push(`!${dataStorePath}`);
-			}
+
 			server.watcher.add(dataStorePath);
 
+			function invalidateDataStore() {
+				const module = server.moduleGraph.getModuleById(RESOLVED_DATA_STORE_VIRTUAL_ID);
+				if (module) {
+					server.moduleGraph.invalidateModule(module);
+				}
+				server.ws.send({
+					type: 'full-reload',
+					path: '*',
+				});
+			}
+
+			// If the datastore file changes, invalidate the virtual module
+
+			server.watcher.on('add', (addedPath) => {
+				if (addedPath === dataStorePath) {
+					invalidateDataStore();
+				}
+			});
+
 			server.watcher.on('change', (changedPath) => {
-				// If the datastore file changes, invalidate the virtual module
 				if (changedPath === dataStorePath) {
-					const module = server.moduleGraph.getModuleById(RESOLVED_DATA_STORE_VIRTUAL_ID);
-					if (module) {
-						server.moduleGraph.invalidateModule(module);
-					}
-					server.ws.send({
-						type: 'full-reload',
-						path: '*',
-					});
+					invalidateDataStore();
 				}
 			});
 		},

packages/astro/src/core/build/index.ts

@@ -61,6 +61,9 @@ export default async function build(
 	const logger = createNodeLogger(inlineConfig);
 	const { userConfig, astroConfig } = await resolveConfig(inlineConfig, 'build');
 	telemetry.record(eventCliSession('build', userConfig));
+
+	const settings = await createSettings(astroConfig, fileURLToPath(astroConfig.root));
+
 	if (inlineConfig.force) {
 		if (astroConfig.experimental.contentCollectionCache) {
 			const contentCacheDir = new URL('./content/', astroConfig.cacheDir);
@@ -70,11 +73,9 @@ export default async function build(
 				logger.warn('content', 'content cache cleared (force)');
 			}
 		}
-		await clearContentLayerCache({ astroConfig, logger, fs });
+		await clearContentLayerCache({ settings, logger, fs });
 	}
 
-	const settings = await createSettings(astroConfig, fileURLToPath(astroConfig.root));
-
 	const builder = new AstroBuilder(settings, {
 		...options,
 		logger,

packages/astro/src/core/dev/dev.ts

@@ -6,8 +6,7 @@ import { green } from 'kleur/colors';
 import { gt, major, minor, patch } from 'semver';
 import type * as vite from 'vite';
 import type { AstroInlineConfig } from '../../@types/astro.js';
-import { DATA_STORE_FILE } from '../../content/consts.js';
-import { globalContentLayer } from '../../content/content-layer.js';
+import { getDataStoreFile, globalContentLayer } from '../../content/content-layer.js';
 import { attachContentServerListeners } from '../../content/index.js';
 import { MutableDataStore } from '../../content/mutable-data-store.js';
 import { globalContentConfigObserver } from '../../content/utils.js';
@@ -108,7 +107,7 @@ export default async function dev(inlineConfig: AstroInlineConfig): Promise<DevS
 
 	let store: MutableDataStore | undefined;
 	try {
-		const dataStoreFile = new URL(DATA_STORE_FILE, restart.container.settings.config.cacheDir);
+		const dataStoreFile = getDataStoreFile(restart.container.settings);
 		if (existsSync(dataStoreFile)) {
 			store = await MutableDataStore.fromFile(dataStoreFile);
 		}

packages/astro/src/core/sync/index.ts

@@ -3,9 +3,9 @@ import { performance } from 'node:perf_hooks';
 import { fileURLToPath } from 'node:url';
 import { dim } from 'kleur/colors';
 import { type HMRPayload, createServer } from 'vite';
-import type { AstroConfig, AstroInlineConfig, AstroSettings } from '../../@types/astro.js';
-import { CONTENT_TYPES_FILE, DATA_STORE_FILE } from '../../content/consts.js';
-import { globalContentLayer } from '../../content/content-layer.js';
+import type { AstroInlineConfig, AstroSettings } from '../../@types/astro.js';
+import { CONTENT_TYPES_FILE } from '../../content/consts.js';
+import { getDataStoreFile, globalContentLayer } from '../../content/content-layer.js';
 import { createContentTypesGenerator } from '../../content/index.js';
 import { MutableDataStore } from '../../content/mutable-data-store.js';
 import { getContentPaths, globalContentConfigObserver } from '../../content/utils.js';
@@ -69,11 +69,11 @@ export default async function sync(
  * Clears the content layer and content collection cache, forcing a full rebuild.
  */
 export async function clearContentLayerCache({
-	astroConfig,
+	settings,
 	logger,
 	fs = fsMod,
-}: { astroConfig: AstroConfig; logger: Logger; fs?: typeof fsMod }) {
-	const dataStore = new URL(DATA_STORE_FILE, astroConfig.cacheDir);
+}: { settings: AstroSettings; logger: Logger; fs?: typeof fsMod }) {
+	const dataStore = getDataStoreFile(settings);
 	if (fs.existsSync(dataStore)) {
 		logger.debug('content', 'clearing data store');
 		await fs.promises.rm(dataStore, { force: true });
@@ -95,7 +95,7 @@ export async function syncInternal({
 	force,
 }: SyncOptions): Promise<void> {
 	if (force) {
-		await clearContentLayerCache({ astroConfig: settings.config, logger, fs });
+		await clearContentLayerCache({ settings, logger, fs });
 	}
 
 	const timerStart = performance.now();
@@ -106,7 +106,7 @@ export async function syncInternal({
 			settings.timer.start('Sync content layer');
 			let store: MutableDataStore | undefined;
 			try {
-				const dataStoreFile = new URL(DATA_STORE_FILE, settings.config.cacheDir);
+				const dataStoreFile = getDataStoreFile(settings);
 				if (existsSync(dataStoreFile)) {
 					store = await MutableDataStore.fromFile(dataStoreFile);
 				}

packages/astro/test/content-layer.test.js

@@ -196,7 +196,11 @@ describe('Content Layer', () => {
 		let devServer;
 		let json;
 		before(async () => {
-			devServer = await fixture.startDevServer();
+			devServer = await fixture.startDevServer({ force: true });
+			// Vite may not have noticed the saved data store yet. Wait a little just in case.
+			await fixture.onNextDataStoreChange(1000).catch(() => {
+				// Ignore timeout, because it may have saved before we get here.
+			})
 			const rawJsonResponse = await fixture.fetch('/collections.json');
 			const rawJson = await rawJsonResponse.text();
 			json = devalue.parse(rawJson);
@@ -286,9 +290,7 @@ describe('Content Layer', () => {
 				return JSON.stringify(data, null, 2);
 			});
 
-			// Writes are debounced to 500ms
-			await new Promise((r) => setTimeout(r, 700));
-
+			await fixture.onNextDataStoreChange();
 			const updatedJsonResponse = await fixture.fetch('/collections.json');
 			const updated = devalue.parse(await updatedJsonResponse.text());
 			assert.ok(updated.fileLoader[0].data.temperament.includes('Bouncy'));

packages/astro/test/test-utils.js

@@ -48,6 +48,7 @@ process.env.ASTRO_TELEMETRY_DISABLED = true;
  * @property {() => Promise<App>} loadTestAdapterApp
  * @property {() => Promise<(req: NodeRequest, res: NodeResponse) => void>} loadNodeAdapterHandler
  * @property {() => Promise<void>} onNextChange
+ * @property {(timeout?: number) => Promise<void>} onNextDataStoreChange
  * @property {typeof check} check
  * @property {typeof sync} sync
  * @property {AstroConfig} config
@@ -183,6 +184,27 @@ export async function loadFixture(inlineConfig) {
 			config.server.port = devServer.address.port; // update port
 			return devServer;
 		},
+		onNextDataStoreChange: (timeout = 5000) => {
+			if (!devServer) {
+				return Promise.reject(new Error('No dev server running'));
+			}
+
+			const dataStoreFile = path.join(root, '.astro', 'data-store.json');
+
+			return new Promise((resolve, reject) => {
+				const changeHandler = (fileName) => {
+					if (fileName === dataStoreFile) {
+						devServer.watcher.removeListener('change', changeHandler);
+						resolve();
+					}
+				};
+				devServer.watcher.on('change', changeHandler);
+				setTimeout(() => {
+					devServer.watcher.removeListener('change', changeHandler);
+					reject(new Error('Data store did not update within timeout'));
+				}, timeout);
+			});
+		},
 		config,
 		resolveUrl,
 		fetch: async (url, init) => {