From 50b2f79905cf17cf2e8afd09b99dd5dfa64261a0 Mon Sep 17 00:00:00 2001 From: Emil Lerch Date: Mon, 29 Apr 2024 00:15:04 -0700 Subject: [PATCH] fix overly aggressive L1 cache reuse --- dist/index.js | 9 +++++---- index.js | 9 +++++++++ test.js | 11 +++++++---- versions.js | 20 +++++++++++++------- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/dist/index.js b/dist/index.js index 9ac3d0e..a3bcfd3 100644 --- a/dist/index.js +++ b/dist/index.js @@ -82166,17 +82166,18 @@ var require_versions = __commonJS({ x64: "x86_64" }[arch]; const host = `${resolvedArch}-${resolvedOs}`; - const machIndex = await getJSON({ url: "https://machengine.org/zig/index.json" }); - const availableVersions = Object.keys(machIndex); + const index = version3.includes("mach") ? await getJSON({ url: "https://machengine.org/zig/index.json" }) : await getJSON({ url: "https://ziglang.org/download/index.json" }); + const availableVersions = Object.keys(index); const useVersion = semver2.valid(version3) ? semver2.maxSatisfying(availableVersions.filter((v) => semver2.valid(v)), version3) : null; - const meta = machIndex[useVersion || version3] || (await getJSON({ url: "https://ziglang.org/download/index.json" }))[useVersion || version3]; + const meta = index[useVersion || version3] || (await getJSON({ url: "https://ziglang.org/download/index.json" }))[useVersion || version3]; if (!meta || !meta[host]) { throw new Error(`Could not find version ${useVersion || version3} for platform ${host}`); } const downloadUrl = meta[host].tarball; const fileWithoutFileType = downloadUrl.match(/.*\/(.*)(\.zip|\.tar\..*$)/)[1]; const variantName = path2.basename(meta[host].tarball).replace(`.${ext}`, "").replace(/\+\S*$/, ""); - return { downloadUrl, fileWithoutFileType, variantName, version: useVersion || version3 }; + const versionFromDownloadUrl = variantName.match(/[^-]*-[^-]*-[^-]*-(.*)/)[1]; + return { downloadUrl, fileWithoutFileType, variantName, version: versionFromDownloadUrl }; } __name(resolveVersion2, "resolveVersion"); module2.exports = { diff --git a/index.js b/index.js index 8e2a113..a77c20e 100644 --- a/index.js +++ b/index.js @@ -17,6 +17,12 @@ const TOOL_NAME = 'zig' async function downloadZig (arch, platform, version, useCache = true) { const ext = extForPlatform(platform) + // There are three levels of data here: + // 1. Local cache - this is literally cache already established with the job + // container, and is a pre-unpacked directory + // 2. Runner cache - this is a cache that is outside the job, but still + // on the host that is running jobs + // 3. The canonical source const { downloadUrl, fileWithoutFileType, @@ -26,12 +32,14 @@ async function downloadZig (arch, platform, version, useCache = true) { ? resolveCommit(arch, platform, version) : await resolveVersion(arch, platform, version) + // Check L1 and return on hit const cachedPath = toolCache.find(TOOL_NAME, useVersion) if (cachedPath) { actions.info(`using cached zig install (version ${useVersion}): ${cachedPath}`) return cachedPath } + // Check L2 and return on hit const cacheKey = `${TOOL_NAME}-${variantName}` if (useCache) { const restorePath = path.join(process.env.RUNNER_TOOL_CACHE, TOOL_NAME, useVersion, arch) @@ -43,6 +51,7 @@ async function downloadZig (arch, platform, version, useCache = true) { } } + // Miss on L1 and on L2. Need to go to canonical source actions.info(`no cached version found. downloading zig ${variantName}`) const downloadPath = await toolCache.downloadTool(downloadUrl) const zigPath = ext === 'zip' diff --git a/test.js b/test.js index 16843be..cfb7c8b 100644 --- a/test.js +++ b/test.js @@ -52,25 +52,25 @@ async function test () { downloadUrl: 'https://pkg.machengine.org/zig/zig-macos-aarch64-0.12.0-dev.2063+804cee3b9.tar.xz', fileWithoutFileType: 'zig-macos-aarch64-0.12.0-dev.2063+804cee3b9', variantName: 'zig-macos-aarch64-0.12.0-dev.2063', - version: '2024.1.0-mach' + version: '0.12.0-dev.2063' }) assert.deepEqual(await resolveVersion('x64', 'linux', '2024.3.0-mach'), { downloadUrl: 'https://pkg.machengine.org/zig/zig-linux-x86_64-0.12.0-dev.3180+83e578a18.tar.xz', fileWithoutFileType: 'zig-linux-x86_64-0.12.0-dev.3180+83e578a18', variantName: 'zig-linux-x86_64-0.12.0-dev.3180', - version: '2024.3.0-mach' + version: '0.12.0-dev.3180' }) assert.deepEqual(await resolveVersion('x64', 'win32', '2024.1.0-mach'), { downloadUrl: 'https://pkg.machengine.org/zig/zig-windows-x86_64-0.12.0-dev.2063+804cee3b9.zip', fileWithoutFileType: 'zig-windows-x86_64-0.12.0-dev.2063+804cee3b9', variantName: 'zig-windows-x86_64-0.12.0-dev.2063', - version: '2024.1.0-mach' + version: '0.12.0-dev.2063' }) assert.deepEqual(await resolveVersion('arm64', 'darwin', '2024.3.0-mach'), { downloadUrl: 'https://pkg.machengine.org/zig/zig-macos-aarch64-0.12.0-dev.3180+83e578a18.tar.xz', fileWithoutFileType: 'zig-macos-aarch64-0.12.0-dev.3180+83e578a18', variantName: 'zig-macos-aarch64-0.12.0-dev.3180', - version: '2024.3.0-mach' + version: '0.12.0-dev.3180' }) await assert.doesNotReject(resolveVersion('x64', 'linux', 'mach-latest')) await assert.doesNotReject(resolveVersion('x64', 'win32', 'mach-latest')) @@ -78,6 +78,9 @@ async function test () { await assert.doesNotReject(resolveVersion('x64', 'linux', 'master')) await assert.doesNotReject(resolveVersion('x64', 'win32', 'master')) await assert.doesNotReject(resolveVersion('arm64', 'darwin', 'master')) + + assert.notEqual((await resolveVersion('arm64', 'darwin', 'mach-latest')).version, 'mach-latest') + assert.notEqual((await resolveVersion('arm64', 'darwin', 'master')).version, 'master') } test().catch((error) => { diff --git a/versions.js b/versions.js index c1d6b0a..9e8c2ce 100644 --- a/versions.js +++ b/versions.js @@ -70,8 +70,8 @@ function getJSON (opts) { * * @returns {string} download URL for the version * @returns {string} file without file type (to match the function above) - * @returns {string} variant name for the version - should be used as cache key - * @returns {string} version name - use for display only + * @returns {string} variant name for the version - used as L2 (runner) cache key + * @returns {string} version name - used as the L1 (job) cache key */ async function resolveVersion (arch, platform, version) { const ext = extForPlatform(platform) @@ -91,17 +91,21 @@ async function resolveVersion (arch, platform, version) { const host = `${resolvedArch}-${resolvedOs}` - // const index = await getJSON({ url: 'https://ziglang.org/download/index.json' }) - const machIndex = await getJSON({ url: 'https://machengine.org/zig/index.json' }) + // Mach json is advertised as a superset, but it's not updated on the same + // timeframe as the ZSF index.json. So notably, 'master' will be an old version + // on machengine.org + const index = version.includes('mach') + ? await getJSON({ url: 'https://machengine.org/zig/index.json' }) + : await getJSON({ url: 'https://ziglang.org/download/index.json' }) - const availableVersions = Object.keys(machIndex) + const availableVersions = Object.keys(index) const useVersion = semver.valid(version) ? semver.maxSatisfying(availableVersions.filter((v) => semver.valid(v)), version) : null // The mach index is advertised as a strict superset of the ziglang index, // but we will fall back to the the ziglang index just in case - const meta = machIndex[useVersion || version] || + const meta = index[useVersion || version] || (await getJSON({ url: 'https://ziglang.org/download/index.json' }))[useVersion || version] if (!meta || !meta[host]) { @@ -117,7 +121,9 @@ async function resolveVersion (arch, platform, version) { // it. This is important as it is used as the cache key const variantName = path.basename(meta[host].tarball).replace(`.${ext}`, '').replace(/\+\S*$/, '') - return { downloadUrl, fileWithoutFileType, variantName, version: useVersion || version } + const versionFromDownloadUrl = variantName.match(/[^-]*-[^-]*-[^-]*-(.*)/)[1] + + return { downloadUrl, fileWithoutFileType, variantName, version: versionFromDownloadUrl } } module.exports = {