Update

.github/workflows/update.yml

@@ -18,6 +18,9 @@ jobs:
     - name: Generate todo.txt
       run: |-
         comm -23 <(sort -u data/current.txt) <(awk -F= '{print $1}' data/identified.txt |sort -u) | sort -u > data/todo.txt
+    - name: Generate merged.txt
+      run: |-
+        node update_merged.js
     - name: Commit and push if it changed
       run: |-
         git config user.name "Automated"

data/current.txt

@@ -60,7 +60,6 @@ d52a3a4356bc07964defc204a2a225b783af073c
 59c01e831e431ec9d29fc4129d5a6613fd9ea411
 ed3e38b47b5883c8f31e0cafe4c0202a6b57425e
 7f91633b43118d653072c45702c2b58913dcb47f
-f7ee31c5aa7b11dfcb95ff83cdee45a4ebab80c6
 29d51a8ec74e5f59d75ffd324f9384e483eef8fb
 77e873325d32a68dc845b5ad4b8550336cd1707b
 8e13714ebb0a48630a9b292a542b9001ff5905d2
@@ -2165,14 +2164,10 @@ ba2a959810ed42b040fe6aa3874b7f01b99b71b1
 8a8c11017d9095fecb54240933586acc26985f0a
 87374110aaf888ba666cfb0021c84cf7514fcc7f
 35d5703ba03161a9731bb22d489ebea05961fa36
-cb01c0a34221f05062a8de1d9bb89c752546d534
-54fbf7b93a67cf856abab926cbb8fbc00559f339
 68d8da226dabd07200acbd5f50b9507ab28248f7
 b88c908d8e751d64ab883639f04944f2a365db83
-a66df261120b6c2311c6ef0b1bab4e583afcbcc0
 47f35f2106f6f9c57ab03774d439e2a788ef6e94
 3b07fbf8a3ffcddbce8ea2eabc6ae5eed07056a2
-59734eec0f6b88730d8dd1545bef6c064fa75554
 da9e239d7e0f2940494e266de69cab206ad837f8
 6e503bb6fe14c424c09af15fe5909bf4a74dd18c
 52e4b0e28775ee23720f2113627a5967353e26dd
@@ -2192,7 +2187,6 @@ f70ae6d11639733239d39a8b71050e43d3056039
 e057d482d33223e9cb48aed3e2d51396a19dbd26
 082460ce7c3155de7693f132a10e15087e5ea8b8
 145cec1a2b5213969beb23be3cdce7e907d0128e
-713e683f16a007e2e2235776aa7d8fbc81439f66
 5774418c4f9307c2770fea421fcaad2dba7d1e76
 dca4b0026dbcb1eb9b20b738fcdb462e591a1382
 d13a3817c275d624eb7e9a2f6d166257f637fe02
@@ -2222,8 +2216,6 @@ bf1056b26eefb4a6d3d49cd1ded9d51c66e3d1e5
 c59890015b012943863ccdb2aa9985a5f0623eac
 3f822334f3873fbb80ead8ab29f60765f2ceb840
 e9600bc4a811f2ef98c4e5912bb6d378ab0f1795
-35da760e9672e17b872ae6381cb50a075c4baa58
-464faa661db64d34508a6545da01fc28de775fe5
 4f5fe28273f48a5bd10b777cde9d89be03fa7e22
 76c0612fa881959fa7fcaaec9cd36be50b43c26a
 6fdddfbc9ce0b32147218f18e7d99f1197d76f38
@@ -2281,22 +2273,13 @@ d7e2998bebc052db4cb9787bf0bc5c39d77b97ab
 c2e01427eff99dcc4198f9636167c9ca36c776c6
 31ae9425089d6af8ce209a70b4a6fc4988b06b2a
 75e138204b7f4770e9de091f35131b136d935336
-b3df5eaac6550f73d938d4f21e3e39168834fd79
 ac79554c8640b1e4201476504a6274c19decf490
-8deb6a735dc78da680d21d7e7f9842cb5fd079e1
 c1b5fe526ff87c3161db9310bf8aef82ca17a5a8
 220ea286ded620bc1ed647187b3bd13a3489a8e7
 41144dd00ec7846f8dfd34f1ab7c35133c4bc08b
 23c68856ec20ca996fb0949f2d6763cea61e7e9f
 2acda96691186a065c12f92a6dc4bf239e8a0e8f
 7f7bbb2104a200c111ac024ee1cfae81be34a634
-03e914e7bff14795bbeca556f7170ca2f5fc49b2
-4bd3fa26056e24a44ea26b7fd0b925aebe5b001f
-56c567dd1ecaebda96b661d99413d34e989830cb
-1fb552744e133315ea6a8a688bc11be03c41d19c
-f80248bc8453a191e10971cff922e7bee40b9268
-63ceeb859c87e3842faa6f34e1925549a3d74007
-c17c6e1214c52f58ae355037f7188b8b4108333a
 a71be8cf63f63b0fefc489f68ea8300ab487661f
 a47d957b4ace8d7187a0bf733f8156ed18af1c13
 be829bacbe6388c39370b6bbd0d4baf17eb2db91

data/todo.txt

@@ -1,5 +1,4 @@
 0038dfc85297b388c0057cbf3a41bbd6d4ac930a
-03e914e7bff14795bbeca556f7170ca2f5fc49b2
 04f48bcd793bb4bdb1c67614ff11a09a0ad7e29b
 070e0b321a6622287328cc06c4e79bf1f9fc1d26
 0bd737def35100fe7cdbf1b91975e647517b9229
@@ -17,7 +16,6 @@
 1bc6521ca892d59a5a9d21341d45c9e92b1a38e3
 1c8394a67ad42b4df9371c95f165886ad71b63cf
 1cfabbfae3ca713b095deefb55e4f6e70780ad75
-1fb552744e133315ea6a8a688bc11be03c41d19c
 1fd47274eed27ddcc7a833b8d756602efdfdc793
 20849a033b9af935b559c38440ef26e22f80fb2e
 20e00be39025eabb300d8b1d8e5defe975a98d5f
@@ -53,7 +51,6 @@
 4872514a33e0b45050f097197227de18b2686d12
 4884e0783d346cd1706a1c935a57941a9d3bf483
 4911907e17b812fd128bc5376516183b88796ff7
-4bd3fa26056e24a44ea26b7fd0b925aebe5b001f
 4e33dba45ea6dd8d226ef1dc7f5990e2faf0e21c
 4f5fe28273f48a5bd10b777cde9d89be03fa7e22
 4f73eaf9aa3c84c2817f73addf2027b99656ffd3
@@ -64,16 +61,13 @@
 5200be7b7b92a515b98b8f1afa4774f83f316624
 53882946fd80fa01af7ff56c1809042fd8b64d13
 5433d68462721d54a56ff33dec7e702dc1d6e308
-54fbf7b93a67cf856abab926cbb8fbc00559f339
 55c862b77be2fe5dfa7f070ce8399b08e6346bcf
-56c567dd1ecaebda96b661d99413d34e989830cb
 5774418c4f9307c2770fea421fcaad2dba7d1e76
 5b487786b10fc8e1f87aee64e237b1c9394df9e2
 5bec25f1b498dcc60264405663833b5afdc63cea
 5c720a83d03808e4e85fee95939d064d3b71b1d1
 5d64ecee1e2494299dba4f5835e9e49254a3f6b4
 5e09c54b6110efbf5683fca4c3eeb1ff97acc037
-63ceeb859c87e3842faa6f34e1925549a3d74007
 63ebc3b0367ebf1fdcae7748efb2b135140dcd87
 644d719a84dac17295072bbf8dd06c00f0c856ea
 67ebeb430865c4b7bcd080005acb63b681b2842f
@@ -85,7 +79,6 @@
 6d804784427cdc233986d0ae3c3a635f852bb7a6
 6fdddfbc9ce0b32147218f18e7d99f1197d76f38
 70b79fc3437590e36b7f2e439882efffcf752ab6
-713e683f16a007e2e2235776aa7d8fbc81439f66
 722723a67608c6982c859514e8e9e930d80d2656
 749b3db3f8874a2a325d0ebbd0f845304adde4bf
 75af2e3fe08e5e6668884d3a37e6749102177f72
@@ -147,7 +140,6 @@ be5f2a05941e67d35a9d214b08d74c43b9089fd6
 bf1056b26eefb4a6d3d49cd1ded9d51c66e3d1e5
 c077ce8473cbac2c68c5dc104ea9cd5012a977e3
 c1004edda9143aad3ed5b0bef9b9eb710928d589
-c17c6e1214c52f58ae355037f7188b8b4108333a
 c1b5fe526ff87c3161db9310bf8aef82ca17a5a8
 c28f84a1345e85774ddced5ff758350f41ec3f5e
 c2e01427eff99dcc4198f9636167c9ca36c776c6
@@ -189,8 +181,6 @@ f2fa0d8e5c2e4935b1c61e594d741fbe33e5e739
 f4d6d5b04763d885816ef94187fe5e3df8e4737c
 f6e32736267315221eb16d4fa944fd38b7946325
 f7273c2a39d8c824171a415938a51b719502311d
-f7ee31c5aa7b11dfcb95ff83cdee45a4ebab80c6
-f80248bc8453a191e10971cff922e7bee40b9268
 f887b5371bb23915e02a526f8ccbc5fef7066d43
 f958d4f3fda570ea260500d3a8722f7dcc60b6a2
 fa493cc71c28027f14004086f7a034a4495df58d

update_merged.js

@@ -0,0 +1,22 @@
+// for each line in data/current.txt, if it is in data/identified.txt, print it instead of the line in data/current.txt
+// if it is not in data/identified.txt, print the line in data/current.txt
+
+// the usefulness of this is that it allows us to see possible context for unidentified hostnames, to maybe guess what they are
+
+var fs = require('fs');
+var current = fs.readFileSync('data/current.txt', 'utf8');
+var identified = fs.readFileSync('data/identified.txt', 'utf8');
+var currentLines = current.split('\n');
+var identifiedLines = identified.split('\n');
+var outputLines = [];
+currentLines.forEach(function(line) {
+  // if any of the lines in identifiedLines starts with the line in currentLines, print the line in identifiedLines
+  var identifiedLine = identifiedLines.find(function(identifiedLine) {
+    return identifiedLine.startsWith(line);
+  });
+  outputLines.push(identifiedLine ? identifiedLine : line);
+});
+
+// write to data/merged.txt
+fs.writeFileSync('data/merged.txt', outputLines.join('\n') + '\n');
+console.log('wrote ' + outputLines.length + ' lines to data/merged.txt');