diff --git a/package-lock.json b/package-lock.json index d7c570b..53efa3b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -303,9 +303,9 @@ } }, "node_modules/@types/express-serve-static-core": { - "version": "4.17.31", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.31.tgz", - "integrity": "sha512-DxMhY+NAsTwMMFHBTtJFNp5qiHKJ7TeqOo23zVEM9alT1Ml27Q3xcTH0xwxn7Q0BbMcVEJOs/7aQtUWupUQN3Q==", + "version": "4.17.32", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.32.tgz", + "integrity": "sha512-aI5h/VOkxOF2Z1saPy0Zsxs5avets/iaiAJYznQFm5By/pamU31xWKL//epiF4OfUA2qTOc9PV6tCUjhO8wlZA==", "dev": true, "dependencies": { "@types/node": "*", @@ -314,9 +314,9 @@ } }, "node_modules/@types/luxon": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.1.0.tgz", - "integrity": "sha512-gCd/HcCgjqSxfMrgtqxCgYk/22NBQfypwFUG7ZAyG/4pqs51WLTcUzVp1hqTbieDYeHS3WoVEh2Yv/2l+7B0Vg==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.2.0.tgz", + "integrity": "sha512-lGmaGFoaXHuOLXFvuju2bfvZRqxAqkHPx9Y9IQdQABrinJJshJwfNCKV+u7rR3kJbiqfTF/NhOkcxxAFrObyaA==", "dev": true }, "node_modules/@types/lzma-native": { @@ -2787,9 +2787,9 @@ } }, "@types/express-serve-static-core": { - "version": "4.17.31", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.31.tgz", - "integrity": "sha512-DxMhY+NAsTwMMFHBTtJFNp5qiHKJ7TeqOo23zVEM9alT1Ml27Q3xcTH0xwxn7Q0BbMcVEJOs/7aQtUWupUQN3Q==", + "version": "4.17.32", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.32.tgz", + "integrity": "sha512-aI5h/VOkxOF2Z1saPy0Zsxs5avets/iaiAJYznQFm5By/pamU31xWKL//epiF4OfUA2qTOc9PV6tCUjhO8wlZA==", "dev": true, "requires": { "@types/node": "*", @@ -2798,9 +2798,9 @@ } }, "@types/luxon": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.1.0.tgz", - "integrity": "sha512-gCd/HcCgjqSxfMrgtqxCgYk/22NBQfypwFUG7ZAyG/4pqs51WLTcUzVp1hqTbieDYeHS3WoVEh2Yv/2l+7B0Vg==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.2.0.tgz", + "integrity": "sha512-lGmaGFoaXHuOLXFvuju2bfvZRqxAqkHPx9Y9IQdQABrinJJshJwfNCKV+u7rR3kJbiqfTF/NhOkcxxAFrObyaA==", "dev": true }, "@types/lzma-native": { diff --git a/repoconfig.yml b/repoconfig.yml index 523eb6c..12f91bb 100644 --- a/repoconfig.yml +++ b/repoconfig.yml @@ -1,6 +1,7 @@ apt-config: portListen: 3000 enableHash: true + saveFiles: true pgpKey: private: ./private.key public: ./public.key diff --git a/src/apt_repo_v2.ts b/src/express_route.ts similarity index 82% rename from src/apt_repo_v2.ts rename to src/express_route.ts index ca527c2..5cfad60 100644 --- a/src/apt_repo_v2.ts +++ b/src/express_route.ts @@ -1,6 +1,6 @@ -import { DebianPackage, httpRequestGithub, httpRequest, DockerRegistry } from "@sirherobrine23/coreutils"; +import { DebianPackage, httpRequestGithub, httpRequest, DockerRegistry, extendFs } from "@sirherobrine23/coreutils"; import { getConfig, distManegerPackages } from "./repoConfig.js"; -import { watchFile } from "node:fs"; +import { createReadStream, createWriteStream, watchFile, promises as fs } from "node:fs"; import { Readable } from "node:stream"; import { CronJob } from "cron"; import { format } from "node:util"; @@ -254,6 +254,8 @@ export default async function main(configPath: string) { // Loading and update packages const cronJobs: CronJob[] = []; const waitPromises: Promise[] = []; + const saveFile = repositoryConfig["apt-config"]?.saveFiles; + const rootPool = repositoryConfig["apt-config"]?.poolPath; for (const dist in repositoryConfig.repositories) { const targets = repositoryConfig.repositories[dist].targets; for (const repository of targets) { @@ -271,11 +273,18 @@ export default async function main(configPath: string) { repositoryConfig: repository, control, async getStream() { + const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`); + if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool); return new Promise((done, reject) => registry.blobLayerStream(data.layer.digest).then(stream => { stream.on("error", reject); stream.pipe(tar.list({ - onentry(getEntry) { + async onentry(getEntry) { if (getEntry.path !== entry.path) return null; + if (saveFile) { + const mainPath = path.resolve(filePool, ".."); + if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true}); + entry.pipe(createWriteStream(filePool)); + } return done(getEntry as any); } // @ts-ignore @@ -296,8 +305,19 @@ export default async function main(configPath: string) { }))); return Promise.all(release.map(async release => Promise.all(release.assets.map(async ({browser_download_url, name}) => { if (!name.endsWith(".deb")) return null; - const getStream = () => httpRequest.pipeFetch(browser_download_url); - const control = await DebianPackage.extractControl(await getStream()); + const control = await DebianPackage.extractControl(await httpRequest.pipeFetch(browser_download_url)); + const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`); + const getStream = async () => { + if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool); + if (saveFile) { + const mainPath = path.resolve(filePool, ".."); + if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true}); + const fileStream = await httpRequest.pipeFetch(browser_download_url); + fileStream.pipe(createWriteStream(filePool)); + return fileStream; + } + return httpRequest.pipeFetch(browser_download_url); + } return packInfos.addPackage(dist, repository.suite ?? release.tag_name, { repositoryConfig: repository, control, @@ -308,8 +328,19 @@ export default async function main(configPath: string) { const release = await httpRequestGithub.getRelease({owner: repository.owner, repository: repository.repository, token: repository.token, peer: repository.assetsLimit, all: false}); return Promise.all(release.map(async release => Promise.all(release.assets.map(async ({browser_download_url, name}) => { if (!name.endsWith(".deb")) return null; - const getStream = () => httpRequest.pipeFetch(browser_download_url); - const control = await DebianPackage.extractControl(await getStream()); + const control = await DebianPackage.extractControl(await httpRequest.pipeFetch(browser_download_url)); + const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`); + const getStream = async () => { + if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool); + if (saveFile) { + const mainPath = path.resolve(filePool, ".."); + if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true}); + const fileStream = await httpRequest.pipeFetch(browser_download_url); + fileStream.pipe(createWriteStream(filePool)); + return fileStream; + } + return httpRequest.pipeFetch(browser_download_url); + } return packInfos.addPackage(dist, repository.suite ?? release.tag_name, { repositoryConfig: repository, control, @@ -329,9 +360,21 @@ export default async function main(configPath: string) { }); return true; }).filter(({path, type}) => path.endsWith(".deb") && type === "blob"); - return Promise.all(filtedTree.map(async ({path}) => { - const getStream = () => httpRequest.pipeFetch(`https://raw.githubusercontent.com/${repository.owner}/${repository.repository}/${repository.tree}/${path}`); - const control = await DebianPackage.extractControl(await getStream()); + return Promise.all(filtedTree.map(async ({path: filePath}) => { + const downloadUrl = `https://raw.githubusercontent.com/${repository.owner}/${repository.repository}/${repository.tree}/${filePath}`; + const control = await DebianPackage.extractControl(await httpRequest.pipeFetch(downloadUrl)); + const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`); + const getStream = async () => { + if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool); + if (saveFile) { + const mainPath = path.resolve(filePool, ".."); + if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true}); + const fileStream = await httpRequest.pipeFetch(downloadUrl); + fileStream.pipe(createWriteStream(filePool)); + return fileStream; + } + return httpRequest.pipeFetch(downloadUrl); + } return packInfos.addPackage(dist, repository.suite ?? repository.tree, { repositoryConfig: repository, control, @@ -339,7 +382,7 @@ export default async function main(configPath: string) { }); })); } else if (repository.from === "google_drive") { - + } else if (repository.from === "oracle_bucket") { } diff --git a/src/index.ts b/src/index.ts index 0919ea6..3a7c028 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,10 +1,10 @@ #!/usr/bin/env node -import yargs from "yargs"; -import repo from "./apt_repo_v2.js"; -import openpgp from "openpgp"; import { getConfig, saveConfig } from "./repoConfig.js"; -import path from "node:path"; import { writeFile } from "node:fs/promises"; +import openpgp from "openpgp"; +import yargs from "yargs"; +import path from "node:path"; +import repo from "./express_route.js"; yargs(process.argv.slice(2)).version(false).help().demandCommand().strictCommands().alias("h", "help").option("cofig-path", { type: "string", diff --git a/src/mirror.ts b/src/mirror.ts index 2f368a8..8c0306c 100644 --- a/src/mirror.ts +++ b/src/mirror.ts @@ -1,7 +1,7 @@ import { httpRequest, DebianPackage } from "@sirherobrine23/coreutils"; +import { promisify } from "node:util"; import openpgp from "openpgp"; import zlib from "node:zlib"; -import { promisify } from "node:util"; export async function getRelease(uri: string, options: {dist: string}) { let Release = (await httpRequest.bufferFetch(`${uri}/dists/${options.dist}/InRelease`).catch(() => httpRequest.bufferFetch(`${uri}/dists/${options.dist}/Release`))).data.toString("utf8").trim(); diff --git a/src/repoConfig.ts b/src/repoConfig.ts index 1ec40e7..ffb322a 100644 --- a/src/repoConfig.ts +++ b/src/repoConfig.ts @@ -1,11 +1,11 @@ -import coreUtils, { DockerRegistry, extendsCrypto } from "@sirherobrine23/coreutils"; -import { debianControl } from "@sirherobrine23/coreutils/src/deb.js"; -import { Readable, Writable } from "node:stream"; -import { createGzip } from "node:zlib"; +import coreUtils, { DockerRegistry, extendFs, extendsCrypto } from "@sirherobrine23/coreutils"; import { Compressor as lzmaCompressor } from "lzma-native"; +import { Readable, Writable } from "node:stream"; +import { debianControl } from "@sirherobrine23/coreutils/src/deb.js"; +import { createGzip } from "node:zlib"; import yaml from "yaml"; -import fs from "node:fs/promises"; import path from "node:path"; +import fs from "node:fs/promises"; export type apt_config = { origin?: string, @@ -74,7 +74,8 @@ export type repository = ({ export type backendConfig = Partial<{ "apt-config"?: apt_config & { portListen?: number, - rootPath?: string, + poolPath?: string, + saveFiles?: boolean, pgpKey?: { private: string, public: string, @@ -97,7 +98,31 @@ export async function getConfig(filePath: string) { if (!await coreUtils.extendFs.exists(filePath)) throw new Error("config File not exists"); const fixedConfig: backendConfig = {}; const configData: backendConfig = yaml.parse(await fs.readFile(filePath, "utf8")); - fixedConfig["apt-config"] = configData["apt-config"] ?? {enableHash: true, label: "apt-stream"}; + fixedConfig["apt-config"] = {}; + if (configData["apt-config"]) { + const rootData = configData["apt-config"]; + fixedConfig["apt-config"].portListen = rootData.portListen ?? 3000; + fixedConfig["apt-config"].poolPath = rootData.poolPath ?? path.join(process.cwd(), "apt-stream"); + fixedConfig["apt-config"].saveFiles = rootData.saveFiles ?? false; + if (fixedConfig["apt-config"].poolPath && !await extendFs.exists(fixedConfig["apt-config"].poolPath)) await fs.mkdir(fixedConfig["apt-config"].poolPath, {recursive: true}); + if (rootData.codename) fixedConfig["apt-config"].codename = rootData.codename; + if (rootData.origin) fixedConfig["apt-config"].origin = rootData.origin; + if (rootData.label) fixedConfig["apt-config"].label = rootData.label; + if (rootData.enableHash) fixedConfig["apt-config"].enableHash = rootData.enableHash; + if (rootData.sourcesHost) fixedConfig["apt-config"].sourcesHost = rootData.sourcesHost; + if (rootData.pgpKey) { + if (!(rootData.pgpKey.private && rootData.pgpKey.public)) throw new Error("pgpKey not defined"); + const privateKey = rootData.pgpKey.private.startsWith("---") ? rootData.pgpKey.private : await fs.readFile(path.resolve(rootData.pgpKey.private), "utf8"); + const publicKey = rootData.pgpKey.public.startsWith("---") ? rootData.pgpKey.public : await fs.readFile(path.resolve(rootData.pgpKey.public), "utf8"); + let passphrase = rootData.pgpKey.passphrase; + if (!passphrase) passphrase = undefined + fixedConfig["apt-config"].pgpKey = { + private: privateKey, + public: publicKey, + passphrase + }; + } + } if (fixedConfig["apt-config"].pgpKey) { const pgpKey = fixedConfig["apt-config"].pgpKey; if (!pgpKey.private.startsWith("---")) fixedConfig["apt-config"].pgpKey.private = await fs.readFile(path.resolve(path.dirname(filePath), pgpKey.private), "utf8"); @@ -324,7 +349,7 @@ export class distManegerPackages { return Promise.resolve(packageData.getStream()).then(stream => ({control: packageData.control, repository: packageData.repositoryConfig, stream})); } - public async createPackages(options?: {compress?: "gzip" | "xz", writeStream?: Writable, dist?: string, package?: string, arch?: string, suite?: string}) { + public async createPackages(options?: {compress?: "gzip" | "xz", writeStream?: Writable, singlePackages?: boolean, dist?: string, package?: string, arch?: string, suite?: string}) { const distribuition = this.distribuitions; const rawWrite = new Readable({read(){}}); let size = 0, addbreak = false, hash: ReturnType|undefined; @@ -361,6 +386,7 @@ export class distManegerPackages { control["Filename"] = `pool/${dist}/${suite}/${control.Package}/${arch}/${control.Version}/download.deb`; const Data = Object.keys(control).map(key => `${key}: ${control[key]}`); rawWrite.push(Data.join("\n")); + if (options?.singlePackages) break; } } }