utils/node-apt
1
0
Fork 0
Code Issues Pull Requests 1 Actions Releases 6 Wiki Activity

Add mongoDB support #8

Merged
Sirherobrine23 merged 6 commits from db into main 2023-01-11 04:44:07 +00:00
Conversation 0 Commits 6 Files Changed 11 +2479 -43
5 changed files with 2479 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit a2fba76f9b - Show all commits

2304
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

1
package.json
View File

@ -48,6 +48,7 @@
"cron": "^2.1.0",
"express": "^4.18.2",
"lzma-native": "^8.0.6",
"mongodb": "^4.13.0",
"openpgp": "^5.5.0",
"tar": "^6.1.13",
"yaml": "^2.2.1",

20
src/express_route.ts
View File

@ -12,8 +12,8 @@ import path from "node:path";
export default async function main(configPath: string) {
// Load config
const packInfos = new distManegerPackages()
let repositoryConfig = await getConfig(configPath);
const packInfos = new distManegerPackages(repositoryConfig);
// Express app
const app = express();
@ -34,12 +34,12 @@ export default async function main(configPath: string) {
});
// Sources list
app.get(["/source_list", "/sources.list"], (req, res) => {
app.get(["/source_list", "/sources.list"], async (req, res) => {
const remotePath = path.posix.resolve(req.baseUrl + req.path, ".."),
protocol = req.headers["x-forwarded-proto"] ?? req.protocol,
hostname = process.env["RAILWAY_STATIC_URL"] ?? `${req.hostname}:${req.socket.localPort}`,
host = repositoryConfig["apt-config"]?.sourcesHost ?? `${protocol}://${hostname}${remotePath}`,
concatPackage = packInfos.getAllDistribuitions(),
concatPackage = await packInfos.getAllDistribuitions(),
type = req.query.type ?? req.query.t,
Conflicting = !!(req.query.conflicting ?? req.query.c);
if (type === "json") {
@ -54,12 +54,12 @@ export default async function main(configPath: string) {
});
// Download
app.get(["/pool", "/"], (_req, res) => res.json(packInfos.getAllDistribuitions()));
app.get("/pool/:dist", (req, res) => res.json(packInfos.getDistribuition(req.params.dist)));
app.get("/pool/:dist/:suite", ({params: {dist, suite}}, res) => res.json(packInfos.getPackageInfo({dist, suite})));
app.get("/pool/:dist/:suite/:arch", ({params: {dist, suite, arch}}, res) => res.json(packInfos.getPackageInfo({dist, suite, arch})));
app.get("/pool/:dist/:suite/:arch/:packageName", ({params: {dist, suite, arch, packageName}}, res) => res.json(packInfos.getPackageInfo({dist, suite, arch, packageName})));
app.get("/pool/:dist/:suite/:arch/:packageName/:version", ({params: {dist, suite, arch, packageName, version}}, res) => res.json(packInfos.getPackageInfo({dist, suite, arch, packageName, version})));
app.get(["/pool", "/"], async (_req, res) => res.json(await packInfos.getAllDistribuitions()));
app.get("/pool/:dist", async (req, res) => res.json(await packInfos.getDistribuition(req.params.dist)));
app.get("/pool/:dist/:suite", async ({params: {dist, suite}}, res) => res.json(await packInfos.getPackageInfo({dist, suite})));
app.get("/pool/:dist/:suite/:arch", async ({params: {dist, suite, arch}}, res) => res.json(await packInfos.getPackageInfo({dist, suite, arch})));
app.get("/pool/:dist/:suite/:arch/:packageName", async ({params: {dist, suite, arch, packageName}}, res) => res.json(await packInfos.getPackageInfo({dist, suite, arch, packageName})));
app.get("/pool/:dist/:suite/:arch/:packageName/:version", async ({params: {dist, suite, arch, packageName, version}}, res) => res.json(await packInfos.getPackageInfo({dist, suite, arch, packageName, version})));
app.get("/pool/:dist/:suite/:arch/:packageName/:version/download.deb", async ({params: {dist, suite, arch, packageName, version}}, res, next) => packInfos.getPackageStream(dist, suite, arch, packageName, version).then(data => data.stream.pipe(res.writeHead(200, {"Content-Type": "application/x-debian-package", "Content-Length": data.control.Size, "Content-Disposition": `attachment; filename="${packageName}_${version}_${arch}.deb"`, "SHA256_hash": data.control.SHA256, "MD5Sum_hash": data.control.MD5sum}))).catch(next));
app.get("/dists/(./)?:dist/:suite/binary-:arch/Packages(.(xz|gz)|)", (req, res) => {
@ -99,7 +99,7 @@ export default async function main(configPath: string) {
// Release
async function createReleaseV1(dist: string) {
const { suites, archs } = packInfos.getDistribuition(dist);
const { suites, archs } = await packInfos.getDistribuition(dist);
const distConfig = repositoryConfig.repositories[dist];
if (!distConfig) throw new Error("Dist not found");
const ReleaseLines = [];

1
src/mirror.ts
View File

@ -82,7 +82,6 @@ export async function getPackages(uri: string, options: {dist: string, suite?: s
const hashs: {file: string, Package: DebianPackage.debianControl}[] = [];
async function addPackages(dist: string, file: string, fn?: (data: DebianPackage.debianControl) => void) {
const urlRequest = `${uri}/dists/${dist}/${file}`;
console.log(`Requesting ${urlRequest}`);
await new Promise<void>(async (done, reject) => {
const stream = (urlRequest.endsWith(".gz")||urlRequest.endsWith(".xz")) ? (await httpRequest.pipeFetch(urlRequest)).pipe(urlRequest.endsWith(".gz") ? zlib.createGunzip() : lzmaDecompressor()) : await httpRequest.pipeFetch(urlRequest);
stream.on("error", (err: any) => {

196
src/repoConfig.ts
View File

@ -1,12 +1,15 @@
import coreUtils, { DebianPackage, DockerRegistry, extendFs, extendsCrypto, httpRequest } from "@sirherobrine23/coreutils";
import { createReadStream, createWriteStream } from "node:fs";
import { MongoClient, ServerApiVersion } from "mongodb";
import { Compressor as lzmaCompressor } from "lzma-native";
import { Readable, Writable } from "node:stream";
import { debianControl } from "@sirherobrine23/coreutils/src/deb.js";
import { createGzip } from "node:zlib";
import { format } from "node:util";
import yaml from "yaml";
import path from "node:path";
import fs from "node:fs/promises";
import { format } from "node:util";
import tar from "tar";
export type apt_config = {
origin?: string,
@ -81,6 +84,11 @@ export type backendConfig = Partial<{
private: string,
public: string,
passphrase?: string
},
mongodb?: {
uri: string,
db?: string,
collection?: string,
}
},
repositories: {
@ -155,6 +163,14 @@ export async function getConfig(config: string) {
passphrase
};
}
if (rootData.mongodb) {
if (!rootData.mongodb.uri) throw new Error("mongodb.uri not defined");
fixedConfig["apt-config"].mongodb = {
uri: rootData.mongodb.uri,
db: rootData.mongodb.db ?? "apt-stream",
collection: rootData.mongodb.collection ?? "packages"
};
}
}
if (fixedConfig["apt-config"].pgpKey) {
const pgpKey = fixedConfig["apt-config"].pgpKey;
@ -288,17 +304,119 @@ type distObject = {
}
}
type dbDist = {
control: debianControl,
repositoryConfig?: repository,
dist: string,
suite: string,
getfile: repository & {file: string, blobLayer?: string}
};
export class distManegerPackages {
public distribuitions: distObject = {};
public addDistribuition(distribuition: string) {
public mongoClinte?: MongoClient;
public internalDist: distObject = {};
public config: backendConfig;
constructor(config: backendConfig) {
this.config = config;
if (config["apt-config"]?.mongodb) {
this.mongoClinte = new MongoClient(config["apt-config"]?.mongodb?.uri, {
serverApi: ServerApiVersion.v1,
});
this.mongoClinte.connect();
}
}
async getPackages(dist?: string, suite?: string): Promise<distObject> {
let repo: distObject = {};
if (!this.mongoClinte) repo = this.distribuitions;
else {
if (dist && typeof dist !== "string") throw new Error("dist must be a string");
if (suite && typeof suite !== "string") throw new Error("suite must be a string");
const saveFile = this.config["apt-config"]?.saveFiles;
const rootPool = this.config["apt-config"]?.poolPath;
const collection = this.mongoClinte.db(this.config["apt-config"]?.mongodb?.db ?? "packages").collection<dbDist>(this.config["apt-config"]?.mongodb.collection ?? "packages");
for (const dataDB of await collection.find({dist, suite}).toArray()) {
const repository = dataDB.getfile, control = dataDB.control;
if (dist && dist !== dataDB.dist) continue;
if (suite && suite !== dataDB.suite) continue;
async function getStream() {
if (repository.from === "mirror") {
const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
return coreUtils.httpRequest.pipeFetch(dataDB.getfile.file);
} else if (repository.from === "oci") {
const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
const registry = await coreUtils.DockerRegistry(repository.image, repository.platfom_target);
return new Promise<Readable>((done, reject) => registry.blobLayerStream(dataDB.getfile.blobLayer).then(stream => {
stream.on("error", reject);
stream.pipe(tar.list({
async onentry(getEntry) {
if (getEntry.path !== dataDB.getfile.file) return null;
if (saveFile) {
const mainPath = path.resolve(filePool, "..");
if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
getEntry.pipe(createWriteStream(filePool));
}
return done(getEntry as any);
}
// @ts-ignore
}).on("error", reject));
}).catch(reject));
} else if (repository.from === "github_release" || repository.from === "github_tree") {
const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
return coreUtils.httpRequest.pipeFetch({
url: dataDB.getfile.file,
headers: (repository.token ? {"Authorization": `token ${repository.token}`} : {})
})
} else if (repository.from === "google_drive") {
const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
const client_id = repository.appSettings.client_id;
const client_secret = repository.appSettings.client_secret;
const token = repository.appSettings.token;
const googleDriver = await coreUtils.googleDriver.GoogleDriver(client_id, client_secret, {
token,
async authCallback(url, token) {
if (url) console.log("Please visit this url to auth google driver: %s", url);
else console.log("Google driver auth success, please save token to config file, token: %s", token);
},
});
return googleDriver.getFileStream(dataDB.getfile.file);
} else if (repository.from === "oracle_bucket") {
const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
const oracleBucket = await coreUtils.oracleBucket(repository.region as any, repository.bucketName, repository.bucketNamespace, repository.auth);
return oracleBucket.getFileStream(dataDB.getfile.file);
}
return null;
}
if (!repo[dataDB.dist]) repo[dataDB.dist] = {};
if (!repo[dataDB.dist][dataDB.suite]) repo[dataDB.dist][dataDB.suite] = {};
if (!repo[dataDB.dist][dataDB.suite][control.Architecture]) repo[dataDB.dist][dataDB.suite][control.Architecture] = [];
repo[dataDB.dist][dataDB.suite][control.Architecture].push({
getStream,
repositoryConfig: dataDB.repositoryConfig,
control,
});
console.log("add package %s %s %s %s", dataDB.dist, dataDB.suite, control.Architecture, control.Package);
};
}
return repo;
}
public async addDistribuition(distribuition: string) {
if (!this.distribuitions[distribuition]) this.distribuitions[distribuition] = {};
return this.distribuitions[distribuition];
}
public addSuite(distribuition: string, suite: string) {
public async addSuite(distribuition: string, suite: string) {
if (!this.distribuitions[distribuition][suite]) this.distribuitions[distribuition][suite] = {};
return this.distribuitions[distribuition][suite];
}
public addArch(distribuition: string, suite: string, arch: string) {
public async addArch(distribuition: string, suite: string, arch: string) {
if (!this.distribuitions[distribuition][suite][arch]) this.distribuitions[distribuition][suite][arch] = [];
return this.distribuitions[distribuition][suite][arch];
}
@ -313,20 +431,39 @@ export class distManegerPackages {
* @param getStream
* @returns
*/
public addPackage(distribuition: string, suite: string, packageData: packageData) {
public async addPackage(distribuition: string, suite: string, packageData: packageData) {
const dist = await this.getPackages(distribuition, suite);
this.addDistribuition(distribuition);
this.addSuite(distribuition, suite);
this.addArch(distribuition, suite, packageData.control.Architecture);
const currentPackages = this.distribuitions[distribuition][suite][packageData.control.Architecture];
if (currentPackages.some(pkg => pkg.control.Package === packageData.control.Package)) {
if (currentPackages.some(pkg => pkg.control.Version === packageData.control.Version && pkg.control.Package === packageData.control.Package)) {
const index = currentPackages.findIndex(pkg => pkg.control.Version === packageData.control.Version && pkg.control.Package === packageData.control.Package);
console.info("[INFO]: Replace %s, with version %s, target arch %s, index number %f", packageData.control.Package, packageData.control.Version, packageData.control.Architecture, index);
return this.distribuitions[distribuition][suite][packageData.control.Architecture][index] = packageData;
const currentPackages = dist[distribuition]?.[suite]?.[packageData.control.Architecture] ?? [];
if (!this.mongoClinte) {
if (currentPackages.some(pkg => pkg.control.Package === packageData.control.Package)) {
if (currentPackages.some(pkg => pkg.control.Version === packageData.control.Version && pkg.control.Package === packageData.control.Package)) {
const index = currentPackages.findIndex(pkg => pkg.control.Version === packageData.control.Version && pkg.control.Package === packageData.control.Package);
return dist[distribuition][suite][packageData.control.Architecture][index] = packageData;
}
}
dist[distribuition][suite][packageData.control.Architecture].push(packageData);
} else {
const collection = this.mongoClinte.db(this.config["apt-config"]?.mongodb?.db ?? "packages").collection(this.config["apt-config"]?.mongodb.collection ?? "packages");
const currentPackages = await collection.findOne({
dist: distribuition,
suite,
cotrol: {
Package: packageData.control.Package,
Version: packageData.control.Version,
Architecture: packageData.control.Architecture,
}
});
if (currentPackages) await collection.deleteOne(currentPackages);
await collection.insertOne({
dist: distribuition,
suite,
control: packageData.control,
repository: packageData.repositoryConfig
});
}
console.info("[INFO]: Add %s, with version %s, target arch %s", packageData.control.Package, packageData.control.Version, packageData.control.Architecture);
this.distribuitions[distribuition][suite][packageData.control.Architecture].push(packageData);
return packageData;
}
@ -341,9 +478,9 @@ export class distManegerPackages {
return data;
}
public getDistribuition(distName: string) {
const dist = this.distribuitions[distName];
if (!dist) throw new Error("Distribuition not exists");
public async getDistribuition(distName: string) {
const dist = (await this.getPackages(distName))[distName];
if (!dist) return null;
const suites = Object.keys(dist);
const suiteData = suites.map(suite => {
const Packages = Object.keys(dist[suite]).map(arch => dist[suite][arch].map(packageInfo => packageInfo.control)).flat();
@ -362,21 +499,23 @@ export class distManegerPackages {
};
}
public getAllDistribuitions() {
return Object.keys(this.distribuitions).map(dist => this.getDistribuition(dist)).flat();
public async getAllDistribuitions() {
const dist = await this.getPackages();
return (await Promise.all(Object.keys(dist).map(dist => this.getDistribuition(dist)))).flat().filter(Boolean);
}
public getPackageInfo(info: {dist: string, suite?: string, arch?: string, packageName?: string, version?: string}) {
public async getPackageInfo(info: {dist: string, suite?: string, arch?: string, packageName?: string, version?: string}) {
const packageDateObject: {[k: string]: {[l: string]: {[a: string]: DebianPackage.debianControl[]}}} = {};
for (const dist in this.distribuitions) {
const distData = await this.getPackages(info.dist, info.suite);
for (const dist in distData) {
if (info.dist && info.dist !== dist) continue;
packageDateObject[dist] = {};
for (const suite in this.distribuitions[dist]) {
for (const suite in distData[dist]) {
if (info.suite && info.suite !== suite) continue;
packageDateObject[dist][suite] = {};
for (const arch in this.distribuitions[dist][suite]) {
for (const arch in distData[dist][suite]) {
if (info.arch && info.arch !== arch) continue;
packageDateObject[dist][suite][arch] = this.distribuitions[dist][suite][arch].map(pkg => pkg.control).filter(pkg => (!info.packageName || pkg.Package === info.packageName) && (!info.version || pkg.Version === info.version));
packageDateObject[dist][suite][arch] = distData[dist][suite][arch].map(pkg => pkg.control).filter(pkg => (!info.packageName || pkg.Package === info.packageName) && (!info.version || pkg.Version === info.version));
}
}
}
@ -397,16 +536,17 @@ export class distManegerPackages {
}
public async getPackageStream(distribuition: string, suite: string, arch: string, packageName: string, version: string) {
if (!this.distribuitions[distribuition]) throw new Error("Distribuition not exists");
if (!this.distribuitions[distribuition][suite]) throw new Error("Suite not exists");
if (!this.distribuitions[distribuition][suite][arch]) throw new Error("Arch not exists");
const packageData = this.distribuitions[distribuition][suite][arch].find(pkg => pkg.control.Package === packageName && pkg.control.Version === version);
const dist = (await this.getPackages(distribuition))[distribuition];
if (!dist) throw new Error("Distribuition not exists");
if (!dist[suite]) throw new Error("Suite not exists");
if (!dist[suite][arch]) throw new Error("Arch not exists");
const packageData = dist[suite][arch].find(pkg => pkg.control.Package === packageName && pkg.control.Version === version);
if (!packageData) throw new Error("Package not exists");
return Promise.resolve(packageData.getStream()).then(stream => ({control: packageData.control, repository: packageData.repositoryConfig, stream}));
}
public async createPackages(options?: {compress?: "gzip" | "xz", writeStream?: Writable, singlePackages?: boolean, dist?: string, package?: string, arch?: string, suite?: string}) {
const distribuition = this.distribuitions;
const distribuition = await this.getPackages(options?.dist);
const rawWrite = new Readable({read(){}});
let size = 0, addbreak = false, hash: ReturnType<typeof extendsCrypto.createHashAsync>|undefined;
if (options?.compress === "gzip") {