Mirrors/browsertrix-crawler
1
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-12-24 11:00:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix --overwrite CLI flag (#220)

Browse Source 
* Delete collection if --overwrite before wb-manager init

* Add tests
This commit is contained in:
Tessa Walsh
2023-02-03 00:02:47 -05:00
committed by GitHub
parent 10e61d4c85
commit 0cf6219d80
2 changed files with 40 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
crawler.js
View File

@@ -202,6 +202,15 @@ export class Crawler {
async bootstrap() {
const logs = path.join(this.collDir, "logs");
if (this.params.overwrite) {
this.logger.info(`Clearing ${this.collDir} before starting`);
try {
fs.rmSync(this.collDir, { recursive: true, force: true });
} catch(e) {
this.logger.error(`Unable to clear ${this.collDir}`, e);
}
}
const initRes = child_process.spawnSync("wb-manager", ["init", this.params.collection], {cwd: this.params.cwd});
if (initRes.status) {
@@ -236,15 +245,6 @@ export class Crawler {
subprocesses.push(child_process.spawn("redis-server", {cwd: "/tmp/", stdio: redisStdio}));
if (this.params.overwrite) {
this.logger.info(`Clearing ${this.collDir} before starting`);
try {
fs.rmSync(this.collDir, { recursive: true, force: true });
} catch(e) {
this.logger.error(`Unable to clear ${this.collDir}`, e);
}
}
opts.env = {...process.env, COLL: this.params.collection, ROLLOVER_SIZE: this.params.rolloverSize};
subprocesses.push(child_process.spawn("uwsgi", [new URL("uwsgi.ini", import.meta.url).pathname], opts));

31
tests/crawl_overwrite.js Normal file
View File

@@ -0,0 +1,31 @@
import child_process from "child_process";
import fs from "fs";
test("ensure --overwrite with existing collection results in a successful crawl", async () => {
child_process.execSync("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ --collection overwrite");
child_process.execSync("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ --collection overwrite --overwrite");
});
test("check that the pages.jsonl file exists in the collection under the pages folder", () => {
expect(fs.existsSync("test-crawls/collections/overwrite/pages/pages.jsonl")).toBe(true);
});
test("check that the WACZ file exists in the collection", () => {
expect(fs.existsSync("test-crawls/collections/overwrite/pages/pages.jsonl")).toBe(true);
});
//-----------
test("ensure --overwrite results in a successful crawl even if collection didn't exist", async () => {
child_process.execSync("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ --collection overwrite-nothing --overwrite");
});
test("check that the pages.jsonl file exists in the collection under the pages folder", () => {
expect(fs.existsSync("test-crawls/collections/overwrite-nothing/pages/pages.jsonl")).toBe(true);
});
test("check that the WACZ file exists in the collection", () => {
expect(fs.existsSync("test-crawls/collections/overwrite-nothing/pages/pages.jsonl")).toBe(true);
});