From c73e9ce30c28f8be2ebbdc8895096e317f62070b Mon Sep 17 00:00:00 2001
From: Ilya Kreymer <ikreymer@gmail.com>
Date: Fri, 19 Dec 2025 22:18:01 -0800
Subject: [PATCH] remove extra sleep

---
 src/indexer.ts    | 4 ----
 src/util/state.ts | 2 +-
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/indexer.ts b/src/indexer.ts
index 6a466e97..993ef23c 100644
--- a/src/indexer.ts
+++ b/src/indexer.ts
@@ -9,7 +9,6 @@ import { initRedisWaitForSuccess } from "./util/redis.js";
 import { AsyncIterReader } from "warcio";
 import { RedisDedupeIndex } from "./util/state.js";
 import { basename } from "node:path";
-import { sleep } from "./util/timing.js";
 
 export type DedupeIndexEntry = {
   name: string;
@@ -138,7 +137,6 @@ export class CrawlIndexer {
     }
 
     logger.info("Done!");
-    await sleep(30);
     await dedupeIndex.markImportFinishedTS();
     process.exit(ExitCodes.Success);
   }
@@ -198,7 +196,6 @@ export class CrawlIndexer {
         if (res && res.size) {
           await dedupeIndex.addStats(res.size - size, crawlId, commitToAllkey);
         } else {
-          console.log("NO DUPE", hash, res);
           await dedupeIndex.addRevisitSize(hash, size, crawlId);
         }
       } else if (url && date && hash) {
@@ -210,7 +207,6 @@ export class CrawlIndexer {
           crawlId,
           commitToAllkey,
         );
-        console.log("MATCH DUPE", hash, size);
         await dedupeIndex.matchRevisitSize(hash, size, crawlId, commitToAllkey);
       } else {
         logger.warn("Skipping invalid CDXJ, data missing", {
diff --git a/src/util/state.ts b/src/util/state.ts
index 8b6dc04a..ed23db4c 100644
--- a/src/util/state.ts
+++ b/src/util/state.ts
@@ -401,7 +401,7 @@ export class RedisDedupeIndex {
         const { size, crawlId } = JSON.parse(res);
         await this.addStats(origSize - size, crawlId, commitToAllKey);
       } catch (e) {
-        console.log(e);
+        logger.debug("Error adding revisit size", e, "state");
         // ignore
       }
     }