Mirrors/browsertrix-crawler
1
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-12-24 11:00:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix --overwrite CLI flag (#220)

Browse Source 
* Delete collection if --overwrite before wb-manager init

* Add tests
This commit is contained in:
Tessa Walsh
2023-02-03 00:02:47 -05:00
committed by GitHub
parent 10e61d4c85
commit 0cf6219d80
2 changed files with 40 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
tests/crawl_overwrite.js Normal file
View File

@@ -0,0 +1,31 @@
import child_process from "child_process";
import fs from "fs";
test("ensure --overwrite with existing collection results in a successful crawl", async () => {
child_process.execSync("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ --collection overwrite");
child_process.execSync("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ --collection overwrite --overwrite");
});
test("check that the pages.jsonl file exists in the collection under the pages folder", () => {
expect(fs.existsSync("test-crawls/collections/overwrite/pages/pages.jsonl")).toBe(true);
});
test("check that the WACZ file exists in the collection", () => {
expect(fs.existsSync("test-crawls/collections/overwrite/pages/pages.jsonl")).toBe(true);
});
//-----------
test("ensure --overwrite results in a successful crawl even if collection didn't exist", async () => {
child_process.execSync("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ --collection overwrite-nothing --overwrite");
});
test("check that the pages.jsonl file exists in the collection under the pages folder", () => {
expect(fs.existsSync("test-crawls/collections/overwrite-nothing/pages/pages.jsonl")).toBe(true);
});
test("check that the WACZ file exists in the collection", () => {
expect(fs.existsSync("test-crawls/collections/overwrite-nothing/pages/pages.jsonl")).toBe(true);
});