Mirrors/browsertrix-crawler
1
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-12-24 11:00:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Prettier to the repo, and format all the files! (#428)

Browse Source 
This adds prettier to the repo, and sets up the pre-commit hook to
auto-format as well as lint.
Also updates ignores files to exclude crawls, test-crawls, scratch, dist as needed.
This commit is contained in:
Emma Segal-Grossman
2023-11-09 19:11:11 -05:00
committed by GitHub
parent af1e0860e4
commit 2a49406df7
70 changed files with 3192 additions and 2026 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
tests/crawl_overwrite.js
View File

@@ -1,31 +1,48 @@
import child_process from "child_process";
import fs from "fs";
test("ensure --overwrite with existing collection results in a successful crawl", async () => {
child_process.execSync("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ --collection overwrite");
child_process.execSync(
"docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ --collection overwrite",
);
child_process.execSync("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ --collection overwrite --overwrite");
child_process.execSync(
"docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ --collection overwrite --overwrite",
);
});
test("check that the pages.jsonl file exists in the collection under the pages folder", () => {
expect(fs.existsSync("test-crawls/collections/overwrite/pages/pages.jsonl")).toBe(true);
expect(
fs.existsSync("test-crawls/collections/overwrite/pages/pages.jsonl"),
).toBe(true);
});
test("check that the WACZ file exists in the collection", () => {
expect(fs.existsSync("test-crawls/collections/overwrite/pages/pages.jsonl")).toBe(true);
expect(
fs.existsSync("test-crawls/collections/overwrite/pages/pages.jsonl"),
).toBe(true);
});
//-----------
test("ensure --overwrite results in a successful crawl even if collection didn't exist", async () => {
child_process.execSync("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ --collection overwrite-nothing --overwrite");
child_process.execSync(
"docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ --collection overwrite-nothing --overwrite",
);
});
test("check that the pages.jsonl file exists in the collection under the pages folder", () => {
expect(fs.existsSync("test-crawls/collections/overwrite-nothing/pages/pages.jsonl")).toBe(true);
expect(
fs.existsSync(
"test-crawls/collections/overwrite-nothing/pages/pages.jsonl",
),
).toBe(true);
});
test("check that the WACZ file exists in the collection", () => {
expect(fs.existsSync("test-crawls/collections/overwrite-nothing/pages/pages.jsonl")).toBe(true);
expect(
fs.existsSync(
"test-crawls/collections/overwrite-nothing/pages/pages.jsonl",
),
).toBe(true);
});