文件去重

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
const fs = require("fs");
const path = require("path");
const crypto = require("crypto");

const fileSet = new Set();
const fileList = [];
let count = 0;

function hashFile(path, length) {
const hash = crypto.createHash("sha256");

hash.on("readable", () => {
// 哈希流只会生成一个元素。
const data = hash.read();
if (data) {
const fileHash = data.toString("hex");
if (!fileSet.has(fileHash)) {
fileSet.add(fileHash);
fileList.push(path);
}
count++;
if (count >= length) {
console.log(fileList);
}
}
});

const filedata = fs.readFileSync(path);

hash.write(filedata);
hash.end();
}

fs.readdir(path.resolve("./"), (err, files) => {
const filesTmp = files.filter(file => file.endsWith(".crash"));
filesTmp.forEach(file => {
hashFile(path.join(path.resolve("./"), file), filesTmp.length);
});
});