utils/node-apt
1
0
Fork 0
Code Issues Pull Requests 1 Actions Releases 6 Wiki Activity

Add mongoDB support #8

Merged
Sirherobrine23 merged 6 commits from db into main 2023-01-11 04:44:07 +00:00
Conversation 0 Commits 6 Files Changed 11 +120 -25
5 changed files with 120 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 4c759ca6d4 - Show all commits

36
package-lock.json generated
View File

@ -14,6 +14,7 @@
"lzma-native": "^8.0.6",
"mongodb": "^4.13.0",
"openpgp": "^5.5.0",
"semver": "^7.3.8",
"tar": "^6.1.13",
"yaml": "^2.2.1",
"yargs": "^17.6.2"
@ -25,6 +26,7 @@
"@types/express": "^4.17.15",
"@types/lzma-native": "^4.0.1",
"@types/node": "^18.11.18",
"@types/semver": "^7.3.13",
"@types/tar": "^6.1.3",
"@types/yargs": "^17.0.19",
"ts-node": "^10.9.1",
@ -1414,6 +1416,12 @@
"integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==",
"dev": true
},
"node_modules/@types/semver": {
"version": "7.3.13",
"resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.13.tgz",
"integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==",
"dev": true
},
"node_modules/@types/serve-static": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz",
@ -3204,6 +3212,20 @@
"node": ">=v12.22.7"
}
},
"node_modules/semver": {
"version": "7.3.8",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
"integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
"dependencies": {
"lru-cache": "^6.0.0"
},
"bin": {
"semver": "bin/semver.js"
},
"engines": {
"node": ">=10"
}
},
"node_modules/send": {
"version": "0.18.0",
"resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz",
@ -4966,6 +4988,12 @@
"integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==",
"dev": true
},
"@types/semver": {
"version": "7.3.13",
"resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.13.tgz",
"integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==",
"dev": true
},
"@types/serve-static": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz",
@ -6280,6 +6308,14 @@
"xmlchars": "^2.2.0"
}
},
"semver": {
"version": "7.3.8",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
"integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
"requires": {
"lru-cache": "^6.0.0"
}
},
"send": {
"version": "0.18.0",
"resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz",

2
package.json
View File

@ -38,6 +38,7 @@
"@types/express": "^4.17.15",
"@types/lzma-native": "^4.0.1",
"@types/node": "^18.11.18",
"@types/semver": "^7.3.13",
"@types/tar": "^6.1.3",
"@types/yargs": "^17.0.19",
"ts-node": "^10.9.1",
@ -49,6 +50,7 @@
"lzma-native": "^8.0.6",
"mongodb": "^4.13.0",
"openpgp": "^5.5.0",
"semver": "^7.3.8",
"tar": "^6.1.13",
"yaml": "^2.2.1",
"yargs": "^17.6.2"

81
src/express_route.ts
View File

@ -2,11 +2,14 @@ import { Compressor as lzmaCompress } from "lzma-native";
import { createGzip } from "node:zlib";
import { getConfig } from "./repoConfig.js";
import { Readable } from "node:stream";
import package_maneger from "./packagesData.js";
import package_maneger, { packageSave } from "./packagesData.js";
import coreUtils from "@sirherobrine23/coreutils";
import cluster from "node:cluster";
import express from "express";
import openpgp from "openpgp";
import semver from "semver";
import path from "node:path";
import os from "node:os";
export default async function initApp(config: string) {
const packageConfig = await getConfig(config);
@ -22,6 +25,18 @@ export default async function initApp(config: string) {
res.on("close", () => console.log("[%s, cluserID: %s]: Path: %s, Method: %s, IP: %s, Status: %f", new Date().toISOString(), cluserID, req.path, req.method, req.ip, res.statusCode));
next();
});
// Info
app.get("/", ({res}) => {
return res.json({
cluster: cluster.worker?.id ?? "No clustered",
cpuCores: os.cpus().length || "Unknown CPU core",
hostArch: process.arch,
hostPlatform: process.platform,
nodeVersion: process.version,
});
});
app.get("/pool/:dist/:suite/:package/:arch/:version/download.deb", async ({params: {dist, suite, package: packageName, arch, version}}, res, next) => {
try {
const data = (await packageManeger.getPackages(dist, suite, packageName, arch, version))?.at(-1);
@ -48,23 +63,59 @@ export default async function initApp(config: string) {
return old;
}, {}))).catch(next));
app.get(["/", "/pool"], ({}, res, next) => packageManeger.getPackages().then(data => res.json(data.reduce((old, current) => {
app.get("/pool", ({}, res, next) => packageManeger.getPackages().then(data => res.json(data.reduce((old, current) => {
if (!old[current.dist]) old[current.dist] = {};
if (!old[current.dist][current.suite]) old[current.dist][current.suite] = [];
old[current.dist][current.suite].push(current.control);
return old;
}, {}))).catch(next));
// Package list
app.get("/source(s)?(.list)?", async (req, res, next) => {
try {
const dists = await Promise.all(await packageManeger.getDists().then(data => data.map(dist => packageManeger.getDistInfo(dist).then(data => ({dist, data})))));
res.set("Content-Type", "text/plain");
const remotePath = path.posix.resolve(req.baseUrl + req.path, ".."), protocol = req.headers["x-forwarded-proto"] ?? req.protocol, hostname = process.env["RAILWAY_STATIC_URL"] ?? `${req.hostname}:${req.socket.localPort}`, host = packageConfig["apt-config"]?.sourcesHost ?? `${protocol}://${hostname}${remotePath}`;
const data = [];
for (const {dist, data: {suites}} of dists) data.push(`deb ${host} ${dist} ${suites.join(" ")}`);
return res.send(data.join("\n"));
} catch (err) {return next(err);}
});
// Dists info
app.get("/dists", ({}, res, next) => packageManeger.getDists().then(data => res.json(data)).catch(next));
app.get("/dists/:dist", ({params: {dist}}, res, next) => packageManeger.getDistInfo(dist).then(data => res.json(data)).catch(next));
app.get("/dists/:dist/source(s)?(.list)?", async (req, res, next) => {
const {dist} = req.params;
try {
const data = await packageManeger.getDistInfo(dist);
res.set("Content-Type", "text/plain");
const remotePath = path.posix.resolve(req.baseUrl + req.path, ".."), protocol = req.headers["x-forwarded-proto"] ?? req.protocol, hostname = process.env["RAILWAY_STATIC_URL"] ?? `${req.hostname}:${req.socket.localPort}`, host = packageConfig["apt-config"]?.sourcesHost ?? `${protocol}://${hostname}${remotePath}`;
return res.send(`deb ${host} ${dist} ${data.suites.join(" ")}`);
} catch (err) {return next(err);}
});
// Create Package, Package.gz and Package.xz
async function createPackages(dist: string, suite: string, arch: string) {
if (!await packageManeger.existsDist(dist)) throw new Error("Distribution not exists");
if (!await packageManeger.existsSuite(dist, suite)) throw new Error("Suite not exists");
const packages = (await packageManeger.getPackages(dist, suite, undefined, arch)).concat(arch !== "all" ? await packageManeger.getPackages(dist, suite, undefined, "all") : []);
function createPackages(packages: packageSave[], dist: string, suite: string) {
if (!packages.length) throw new Error("Check is dist or suite have packages");
const reduced = packages.reduce((old, current) => {
if (!old[current.control.Package]) old[current.control.Package] = [];
old[current.control.Package].push(current);
return old;
}, {} as {[packageName: string]: packageSave[]});
packages = [];
Object.keys(reduced).forEach(packageName => {
reduced[packageName] = reduced[packageName].sort((b, a) => {
const aVersion = semver.valid(semver.coerce(a.control.Version) ?? ""), bVersion = semver.valid(semver.coerce(b.control.Version) ?? "");
return semver.compare(aVersion, bVersion);
});
if (!packageConfig["apt-config"]?.packagesOptions?.uniqueVersion) packages.push(...reduced[packageName]);
else {
const at = reduced[packageName].at(0);
if (at) packages.push(at);
}
});
let rawSize = 0, gzipSize = 0, lzmaSize = 0;
const mainReadstream = new Readable({read(){}}), rawSUMs = coreUtils.extendsCrypto.createHashAsync("all", mainReadstream).then(hash => ({size: rawSize, hash}));
const gzip = mainReadstream.pipe(createGzip()), gzipSUMs = coreUtils.extendsCrypto.createHashAsync("all", gzip).then(hash => ({size: gzipSize, hash}));
@ -72,7 +123,6 @@ export default async function initApp(config: string) {
mainReadstream.on("data", data => rawSize += data.length);
gzip.on("data", data => gzipSize += data.length);
lzma.on("data", data => lzmaSize += data.length);
let fist = true;
for (const {control} of packages) {
if (!(control.Size && (control.MD5sum || control.SHA256 || control.SHA1))) continue;
@ -94,12 +144,14 @@ export default async function initApp(config: string) {
};
}
app.get("/dists/(./)?:dist/:suite/binary-:arch/Packages(.(xz|gz)|)", async ({params: {dist, suite, arch}, path: reqPath}, res, next) => createPackages(dist, suite, arch).then(packages => {
if (reqPath.endsWith(".gz")) return packages.gzip.pipe(res);
else if (reqPath.endsWith(".xz")) return packages.lzma.pipe(res);
else return packages.raw.pipe(res);
}).catch(next));
app.get("/dists/(./)?:dist/:suite/binary-:arch/Packages(.(xz|gz)|)", async ({params: {dist, suite, arch}, path: reqPath}, res, next) => {
try {
const packages = createPackages(await packageManeger.getPackages(dist, suite, undefined, arch), dist, suite);
if (reqPath.endsWith(".gz")) return packages.gzip.pipe(res);
else if (reqPath.endsWith(".xz")) return packages.lzma.pipe(res);
else return packages.raw.pipe(res);
} catch (err) {return next(err);}
});
// Release
async function createRelease(dist: string) {
if (!await packageManeger.existsDist(dist)) throw new Error("Dist exists");
@ -140,7 +192,8 @@ export default async function initApp(config: string) {
Release.SHA1 = [];
Release.MD5sum = [];
const files = await Promise.all(Archs.map(async Arch => Promise.all(Components.map(async Component => {
const {SUMs} = await createPackages(dist, Component, Arch);
const archPackages = packagesArray.filter(x => x.control.Architecture === Arch && x.suite === Component);
const {SUMs} = createPackages(archPackages, dist, Component);
return [
{
file: `${Component}/binary-${Arch}/Packages`,

17
src/packagesData.ts
View File

@ -4,10 +4,10 @@ import { MongoClient, ServerApiVersion, Filter } from "mongodb";
import { apt_config, backendConfig, repository } from "./repoConfig.js";
import { getPackages as mirror } from "./mirror.js";
import { Readable } from "node:stream";
import { format } from "node:util";
import cluster from "node:cluster";
import path from "node:path";
import tar from "tar";
import { format } from "node:util";
export type packageSave = {
dist: string,
@ -365,15 +365,12 @@ export default async function packageManeger(config: backendConfig): Promise<pac
function fixPackage(data: packageSave): packageSave {
if (!data.restoreFileStream) throw new Error("cannot restore file stream!");
data.getFileStream = async function getFileStream() {
if (data.restoreFileStream.from === "github_release"|| data.restoreFileStream.from === "github_tree") return coreUtils.httpRequest.pipeFetch(data.restoreFileStream.url);
else if (data.restoreFileStream.from === "google_drive" && data.repository.from === "google_drive") {
const googleDriver = await coreUtils.googleDriver.GoogleDriver(data.repository.appSettings.client_id, data.repository.appSettings.client_secret, {token: data.repository.appSettings.token});
return googleDriver.getFileStream(data.restoreFileStream.fileId);
} else if (data.restoreFileStream.from === "oracle_bucket" && data.repository.from === "oracle_bucket") {
const oracleBucket = await coreUtils.oracleBucket(data.repository.region as any, data.repository.bucketName, data.repository.bucketNamespace, data.repository.auth);
return oracleBucket.getFileStream(data.restoreFileStream.fileName);
} else if (data.restoreFileStream.from === "mirror" && data.repository.from === "mirror") return coreUtils.httpRequest.pipeFetch(data.restoreFileStream.fileUrl);
else if (data.restoreFileStream.from === "oci" && data.repository.from === "oci") {
if (data.restoreFileStream.fileUrl) return coreUtils.httpRequest.pipeFetch(data.restoreFileStream.fileUrl);
if (data.restoreFileStream.from === "google_drive" && data.repository.from === "google_drive") {
const { appSettings } = data.repository;
const googleDrive = await coreUtils.googleDriver.GoogleDriver(appSettings.client_id, appSettings.client_secret, {token: appSettings.token});
return googleDrive.getFileStream(data.restoreFileStream.fileId);
} else if (data.restoreFileStream.from === "oci" && data.repository.from === "oci") {
const oci = await coreUtils.DockerRegistry(data.repository.image);
return new Promise((done, reject) => {
oci.blobLayerStream(data.restoreFileStream.digest).then((stream) => {

9
src/repoConfig.ts
View File

@ -84,6 +84,9 @@ export type backendConfig = Partial<{
collection?: string,
/** On connect to database drop collection to run in empty data */
dropCollention?: boolean
},
packagesOptions?: {
uniqueVersion?: boolean,
}
},
repositories: {
@ -136,7 +139,11 @@ export async function getConfig(config: string) {
if (typeof configData !== "object") throw new Error("Invalid config file");
const fixedConfig: backendConfig = {
"apt-config": {},
"apt-config": {
packagesOptions: {
uniqueVersion: configData["apt-config"]?.packagesOptions?.uniqueVersion ?? false
}
},
repositories: {}
};
if (configData["apt-config"]) {