utils/node-apt
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Releases 6 Wiki Activity

add save files on request's #7

Merged
Sirherobrine23 merged 1 commits from filesSave into main 2023-01-02 23:55:16 +00:00
Conversation 0 Commits 1 Files Changed 6 +106 -36
6 changed files with 106 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
package-lock.json generated
View File

@ -303,9 +303,9 @@
}
},
"node_modules/@types/express-serve-static-core": {
"version": "4.17.31",
"resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.31.tgz",
"integrity": "sha512-DxMhY+NAsTwMMFHBTtJFNp5qiHKJ7TeqOo23zVEM9alT1Ml27Q3xcTH0xwxn7Q0BbMcVEJOs/7aQtUWupUQN3Q==",
"version": "4.17.32",
"resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.32.tgz",
"integrity": "sha512-aI5h/VOkxOF2Z1saPy0Zsxs5avets/iaiAJYznQFm5By/pamU31xWKL//epiF4OfUA2qTOc9PV6tCUjhO8wlZA==",
"dev": true,
"dependencies": {
"@types/node": "*",
@ -314,9 +314,9 @@
}
},
"node_modules/@types/luxon": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.1.0.tgz",
"integrity": "sha512-gCd/HcCgjqSxfMrgtqxCgYk/22NBQfypwFUG7ZAyG/4pqs51WLTcUzVp1hqTbieDYeHS3WoVEh2Yv/2l+7B0Vg==",
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.2.0.tgz",
"integrity": "sha512-lGmaGFoaXHuOLXFvuju2bfvZRqxAqkHPx9Y9IQdQABrinJJshJwfNCKV+u7rR3kJbiqfTF/NhOkcxxAFrObyaA==",
"dev": true
},
"node_modules/@types/lzma-native": {
@ -2787,9 +2787,9 @@
}
},
"@types/express-serve-static-core": {
"version": "4.17.31",
"resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.31.tgz",
"integrity": "sha512-DxMhY+NAsTwMMFHBTtJFNp5qiHKJ7TeqOo23zVEM9alT1Ml27Q3xcTH0xwxn7Q0BbMcVEJOs/7aQtUWupUQN3Q==",
"version": "4.17.32",
"resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.32.tgz",
"integrity": "sha512-aI5h/VOkxOF2Z1saPy0Zsxs5avets/iaiAJYznQFm5By/pamU31xWKL//epiF4OfUA2qTOc9PV6tCUjhO8wlZA==",
"dev": true,
"requires": {
"@types/node": "*",
@ -2798,9 +2798,9 @@
}
},
"@types/luxon": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.1.0.tgz",
"integrity": "sha512-gCd/HcCgjqSxfMrgtqxCgYk/22NBQfypwFUG7ZAyG/4pqs51WLTcUzVp1hqTbieDYeHS3WoVEh2Yv/2l+7B0Vg==",
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.2.0.tgz",
"integrity": "sha512-lGmaGFoaXHuOLXFvuju2bfvZRqxAqkHPx9Y9IQdQABrinJJshJwfNCKV+u7rR3kJbiqfTF/NhOkcxxAFrObyaA==",
"dev": true
},
"@types/lzma-native": {

1
repoconfig.yml
View File

@ -1,6 +1,7 @@
apt-config:
portListen: 3000
enableHash: true
saveFiles: true
pgpKey:
private: ./private.key
public: ./public.key

65
src/apt_repo_v2.ts → src/express_route.ts
View File

@ -1,6 +1,6 @@
import { DebianPackage, httpRequestGithub, httpRequest, DockerRegistry } from "@sirherobrine23/coreutils";
import { DebianPackage, httpRequestGithub, httpRequest, DockerRegistry, extendFs } from "@sirherobrine23/coreutils";
import { getConfig, distManegerPackages } from "./repoConfig.js";
import { watchFile } from "node:fs";
import { createReadStream, createWriteStream, watchFile, promises as fs } from "node:fs";
import { Readable } from "node:stream";
import { CronJob } from "cron";
import { format } from "node:util";
@ -254,6 +254,8 @@ export default async function main(configPath: string) {
// Loading and update packages
const cronJobs: CronJob[] = [];
const waitPromises: Promise<void>[] = [];
const saveFile = repositoryConfig["apt-config"]?.saveFiles;
const rootPool = repositoryConfig["apt-config"]?.poolPath;
for (const dist in repositoryConfig.repositories) {
const targets = repositoryConfig.repositories[dist].targets;
for (const repository of targets) {
@ -271,11 +273,18 @@ export default async function main(configPath: string) {
repositoryConfig: repository,
control,
async getStream() {
const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
return new Promise<Readable>((done, reject) => registry.blobLayerStream(data.layer.digest).then(stream => {
stream.on("error", reject);
stream.pipe(tar.list({
onentry(getEntry) {
async onentry(getEntry) {
if (getEntry.path !== entry.path) return null;
if (saveFile) {
const mainPath = path.resolve(filePool, "..");
if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
entry.pipe(createWriteStream(filePool));
}
return done(getEntry as any);
}
// @ts-ignore
@ -296,8 +305,19 @@ export default async function main(configPath: string) {
})));
return Promise.all(release.map(async release => Promise.all(release.assets.map(async ({browser_download_url, name}) => {
if (!name.endsWith(".deb")) return null;
const getStream = () => httpRequest.pipeFetch(browser_download_url);
const control = await DebianPackage.extractControl(await getStream());
const control = await DebianPackage.extractControl(await httpRequest.pipeFetch(browser_download_url));
const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
const getStream = async () => {
if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
if (saveFile) {
const mainPath = path.resolve(filePool, "..");
if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
const fileStream = await httpRequest.pipeFetch(browser_download_url);
fileStream.pipe(createWriteStream(filePool));
return fileStream;
}
return httpRequest.pipeFetch(browser_download_url);
}
return packInfos.addPackage(dist, repository.suite ?? release.tag_name, {
repositoryConfig: repository,
control,
@ -308,8 +328,19 @@ export default async function main(configPath: string) {
const release = await httpRequestGithub.getRelease({owner: repository.owner, repository: repository.repository, token: repository.token, peer: repository.assetsLimit, all: false});
return Promise.all(release.map(async release => Promise.all(release.assets.map(async ({browser_download_url, name}) => {
if (!name.endsWith(".deb")) return null;
const getStream = () => httpRequest.pipeFetch(browser_download_url);
const control = await DebianPackage.extractControl(await getStream());
const control = await DebianPackage.extractControl(await httpRequest.pipeFetch(browser_download_url));
const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
const getStream = async () => {
if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
if (saveFile) {
const mainPath = path.resolve(filePool, "..");
if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
const fileStream = await httpRequest.pipeFetch(browser_download_url);
fileStream.pipe(createWriteStream(filePool));
return fileStream;
}
return httpRequest.pipeFetch(browser_download_url);
}
return packInfos.addPackage(dist, repository.suite ?? release.tag_name, {
repositoryConfig: repository,
control,
@ -329,9 +360,21 @@ export default async function main(configPath: string) {
});
return true;
}).filter(({path, type}) => path.endsWith(".deb") && type === "blob");
return Promise.all(filtedTree.map(async ({path}) => {
const getStream = () => httpRequest.pipeFetch(`https://raw.githubusercontent.com/${repository.owner}/${repository.repository}/${repository.tree}/${path}`);
const control = await DebianPackage.extractControl(await getStream());
return Promise.all(filtedTree.map(async ({path: filePath}) => {
const downloadUrl = `https://raw.githubusercontent.com/${repository.owner}/${repository.repository}/${repository.tree}/${filePath}`;
const control = await DebianPackage.extractControl(await httpRequest.pipeFetch(downloadUrl));
const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
const getStream = async () => {
if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
if (saveFile) {
const mainPath = path.resolve(filePool, "..");
if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
const fileStream = await httpRequest.pipeFetch(downloadUrl);
fileStream.pipe(createWriteStream(filePool));
return fileStream;
}
return httpRequest.pipeFetch(downloadUrl);
}
return packInfos.addPackage(dist, repository.suite ?? repository.tree, {
repositoryConfig: repository,
control,
@ -339,7 +382,7 @@ export default async function main(configPath: string) {
});
}));
} else if (repository.from === "google_drive") {
} else if (repository.from === "oracle_bucket") {
}

8
src/index.ts
View File

@ -1,10 +1,10 @@
#!/usr/bin/env node
import yargs from "yargs";
import repo from "./apt_repo_v2.js";
import openpgp from "openpgp";
import { getConfig, saveConfig } from "./repoConfig.js";
import path from "node:path";
import { writeFile } from "node:fs/promises";
import openpgp from "openpgp";
import yargs from "yargs";
import path from "node:path";
import repo from "./express_route.js";
yargs(process.argv.slice(2)).version(false).help().demandCommand().strictCommands().alias("h", "help").option("cofig-path", {
type: "string",

2
src/mirror.ts
View File

@ -1,7 +1,7 @@
import { httpRequest, DebianPackage } from "@sirherobrine23/coreutils";
import { promisify } from "node:util";
import openpgp from "openpgp";
import zlib from "node:zlib";
import { promisify } from "node:util";
export async function getRelease(uri: string, options: {dist: string}) {
let Release = (await httpRequest.bufferFetch(`${uri}/dists/${options.dist}/InRelease`).catch(() => httpRequest.bufferFetch(`${uri}/dists/${options.dist}/Release`))).data.toString("utf8").trim();

42
src/repoConfig.ts
View File

@ -1,11 +1,11 @@
import coreUtils, { DockerRegistry, extendsCrypto } from "@sirherobrine23/coreutils";
import { debianControl } from "@sirherobrine23/coreutils/src/deb.js";
import { Readable, Writable } from "node:stream";
import { createGzip } from "node:zlib";
import coreUtils, { DockerRegistry, extendFs, extendsCrypto } from "@sirherobrine23/coreutils";
import { Compressor as lzmaCompressor } from "lzma-native";
import { Readable, Writable } from "node:stream";
import { debianControl } from "@sirherobrine23/coreutils/src/deb.js";
import { createGzip } from "node:zlib";
import yaml from "yaml";
import fs from "node:fs/promises";
import path from "node:path";
import fs from "node:fs/promises";
export type apt_config = {
origin?: string,
@ -74,7 +74,8 @@ export type repository = ({
export type backendConfig = Partial<{
"apt-config"?: apt_config & {
portListen?: number,
rootPath?: string,
poolPath?: string,
saveFiles?: boolean,
pgpKey?: {
private: string,
public: string,
@ -97,7 +98,31 @@ export async function getConfig(filePath: string) {
if (!await coreUtils.extendFs.exists(filePath)) throw new Error("config File not exists");
const fixedConfig: backendConfig = {};
const configData: backendConfig = yaml.parse(await fs.readFile(filePath, "utf8"));
fixedConfig["apt-config"] = configData["apt-config"] ?? {enableHash: true, label: "apt-stream"};
fixedConfig["apt-config"] = {};
if (configData["apt-config"]) {
const rootData = configData["apt-config"];
fixedConfig["apt-config"].portListen = rootData.portListen ?? 3000;
fixedConfig["apt-config"].poolPath = rootData.poolPath ?? path.join(process.cwd(), "apt-stream");
fixedConfig["apt-config"].saveFiles = rootData.saveFiles ?? false;
if (fixedConfig["apt-config"].poolPath && !await extendFs.exists(fixedConfig["apt-config"].poolPath)) await fs.mkdir(fixedConfig["apt-config"].poolPath, {recursive: true});
if (rootData.codename) fixedConfig["apt-config"].codename = rootData.codename;
if (rootData.origin) fixedConfig["apt-config"].origin = rootData.origin;
if (rootData.label) fixedConfig["apt-config"].label = rootData.label;
if (rootData.enableHash) fixedConfig["apt-config"].enableHash = rootData.enableHash;
if (rootData.sourcesHost) fixedConfig["apt-config"].sourcesHost = rootData.sourcesHost;
if (rootData.pgpKey) {
if (!(rootData.pgpKey.private && rootData.pgpKey.public)) throw new Error("pgpKey not defined");
const privateKey = rootData.pgpKey.private.startsWith("---") ? rootData.pgpKey.private : await fs.readFile(path.resolve(rootData.pgpKey.private), "utf8");
const publicKey = rootData.pgpKey.public.startsWith("---") ? rootData.pgpKey.public : await fs.readFile(path.resolve(rootData.pgpKey.public), "utf8");
let passphrase = rootData.pgpKey.passphrase;
if (!passphrase) passphrase = undefined
fixedConfig["apt-config"].pgpKey = {
private: privateKey,
public: publicKey,
passphrase
};
}
}
if (fixedConfig["apt-config"].pgpKey) {
const pgpKey = fixedConfig["apt-config"].pgpKey;
if (!pgpKey.private.startsWith("---")) fixedConfig["apt-config"].pgpKey.private = await fs.readFile(path.resolve(path.dirname(filePath), pgpKey.private), "utf8");
@ -324,7 +349,7 @@ export class distManegerPackages {
return Promise.resolve(packageData.getStream()).then(stream => ({control: packageData.control, repository: packageData.repositoryConfig, stream}));
}
public async createPackages(options?: {compress?: "gzip" | "xz", writeStream?: Writable, dist?: string, package?: string, arch?: string, suite?: string}) {
public async createPackages(options?: {compress?: "gzip" | "xz", writeStream?: Writable, singlePackages?: boolean, dist?: string, package?: string, arch?: string, suite?: string}) {
const distribuition = this.distribuitions;
const rawWrite = new Readable({read(){}});
let size = 0, addbreak = false, hash: ReturnType<typeof extendsCrypto.createHashAsync>|undefined;
@ -361,6 +386,7 @@ export class distManegerPackages {
control["Filename"] = `pool/${dist}/${suite}/${control.Package}/${arch}/${control.Version}/download.deb`;
const Data = Object.keys(control).map(key => `${key}: ${control[key]}`);
rawWrite.push(Data.join("\n"));
if (options?.singlePackages) break;
}
}
}