Scheduled update

Reviewed-on: #1

README.md

@@ -1,106 +1,109 @@
 # PIA Server List
 
-![Servers](https://img.shields.io/badge/servers-5,252-blue) ![Regions](https://img.shields.io/badge/regions-97-blue) ![Last update](https://img.shields.io/badge/last_updated-Mon_Apr_29_06:19:45_UTC_2024-blue)
+![Servers](https://img.shields.io/badge/servers-18,240-blue)
+![Regions](https://img.shields.io/badge/regions-97-blue)
+![Last update](https://img.shields.io/badge/last_updated-Wed_Jul_31_17:54:31_UTC_2024-blue)
 
-This is a list of the OpenVPN servers provided by Private Internet Access (PIA). The list is updated hourly.
+This is a list of the OpenVPN servers provided by Private Internet Access (PIA). The list is updated hourly. </br>
+Servers that have not been active in the last 2 weeks will be automatically removed.
 
 ## Servers
 | Region               | Servers |
 |----------------------|---------|
-| netherlands | 109 |
-| ca_vancouver | 98 |
-| au_melbourne | 98 |
-| ca_montreal | 98 |
-| ca_toronto | 97 |
-| us_chicago | 97 |
-| us_silicon_valley | 97 |
-| us_east | 96 |
-| us_washington_dc | 96 |
-| us_new_york | 95 |
-| uk_london | 95 |
-| us_california | 94 |
-| us_texas | 94 |
-| us_seattle | 93 |
-| us_atlanta | 90 |
-| us_las_vegas | 89 |
-| switzerland | 89 |
-| us_west | 89 |
-| us_florida | 88 |
-| de_frankfurt | 88 |
-| ca_ontario | 86 |
-| us_houston | 86 |
-| uk_southampton | 84 |
-| japan | 83 |
-| de_berlin | 83 |
-| au_sydney | 82 |
-| mexico | 80 |
-| sweden | 79 |
-| uk_manchester | 79 |
-| singapore | 78 |
-| us_denver | 78 |
-| new_zealand | 78 |
-| au_perth | 73 |
-| france | 70 |
-| czech_republic | 64 |
-| norway | 63 |
-| ireland | 62 |
-| estonia | 61 |
-| bahamas | 59 |
-| israel | 59 |
-| spain | 57 |
-| argentina | 55 |
-| brazil | 54 |
-| denmark | 52 |
-| austria | 51 |
-| panama | 50 |
-| italy | 49 |
-| ukraine | 48 |
-| finland | 48 |
-| luxembourg | 46 |
-| latvia | 46 |
-| turkey | 45 |
-| romania | 45 |
-| lithuania | 44 |
-| poland | 42 |
-| vietnam | 41 |
-| slovakia | 41 |
-| greenland | 40 |
-| belgium | 40 |
-| greece | 37 |
-| andorra | 35 |
-| venezuela | 35 |
-| albania | 34 |
-| algeria | 34 |
-| taiwan | 34 |
-| united_arab_emirates | 33 |
-| china | 33 |
-| malta | 32 |
-| philippines | 31 |
-| hungary | 29 |
-| qatar | 29 |
+| ca_montreal | 816 |
+| netherlands | 811 |
+| us_texas | 708 |
+| us_chicago | 702 |
+| ca_toronto | 643 |
+| us_silicon_valley | 601 |
+| us_washington_dc | 589 |
+| us_atlanta | 565 |
+| us_west | 559 |
+| ca_vancouver | 547 |
+| us_las_vegas | 542 |
+| us_new_york | 541 |
+| us_east | 536 |
+| uk_london | 533 |
+| us_california | 526 |
+| us_florida | 496 |
+| us_seattle | 496 |
+| ca_ontario | 430 |
+| au_melbourne | 426 |
+| switzerland | 420 |
+| de_frankfurt | 417 |
+| us_houston | 384 |
+| au_sydney | 355 |
+| uk_southampton | 318 |
+| japan | 308 |
+| de_berlin | 261 |
+| us_denver | 231 |
+| uk_manchester | 224 |
+| singapore | 218 |
+| france | 202 |
+| sweden | 192 |
+| au_perth | 183 |
+| new_zealand | 183 |
+| mexico | 139 |
+| ireland | 132 |
+| romania | 120 |
+| estonia | 117 |
+| norway | 112 |
+| belgium | 102 |
+| israel | 100 |
+| spain | 99 |
+| denmark | 96 |
+| czech_republic | 94 |
+| italy | 87 |
+| finland | 84 |
+| bahamas | 78 |
+| argentina | 75 |
+| austria | 71 |
+| panama | 71 |
+| brazil | 69 |
+| poland | 64 |
+| luxembourg | 62 |
+| latvia | 60 |
+| ukraine | 59 |
+| india | 59 |
+| lithuania | 56 |
+| greenland | 56 |
+| turkey | 52 |
+| slovakia | 52 |
+| andorra | 48 |
+| vietnam | 46 |
+| taiwan | 46 |
+| venezuela | 45 |
+| greece | 45 |
+| portugal | 43 |
+| united_arab_emirates | 38 |
+| china | 38 |
+| malta | 37 |
+| albania | 36 |
+| philippines | 36 |
+| algeria | 36 |
+| qatar | 33 |
+| bangladesh | 31 |
+| hungary | 30 |
+| liechtenstein | 29 |
+| isle_of_man | 29 |
+| saudi_arabia | 28 |
 | bulgaria | 28 |
-| india | 28 |
 | south_africa | 28 |
-| bangladesh | 28 |
-| liechtenstein | 27 |
-| saudi_arabia | 26 |
-| isle_of_man | 26 |
-| portugal | 25 |
-| montenegro | 25 |
+| macao | 26 |
+| montenegro | 26 |
+| nigeria | 25 |
+| moldova | 24 |
+| cambodia | 24 |
 | cyprus | 24 |
-| nigeria | 24 |
-| macao | 24 |
+| morocco | 24 |
+| egypt | 24 |
 | iceland | 24 |
-| moldova | 23 |
-| cambodia | 23 |
-| morocco | 23 |
 | kazakhstan | 23 |
-| egypt | 23 |
 | macedonia | 23 |
-| monaco | 21 |
+| monaco | 22 |
+| georgia | 22 |
 | hong_kong | 20 |
-| georgia | 20 |
-| serbia | 14 |
+| serbia | 15 |
 | armenia | 12 |
 | mongolia | 12 |
 | sri_lanka | 11 |

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "local>Fascinated/renovate-config"
+  ]
+}

src/main/java/cc/fascinated/piaservers/common/GitUtils.java

@@ -13,13 +13,13 @@ public class GitUtils {
     public static void commitFiles(String message, Path... files) {
         System.out.println("Committing files");
         if (Config.isProduction()) {
-            runCommand("git", "config", "--global", "user.email", "liam+pia-servers-ci@fascinated.cc");
-            runCommand("git", "config", "--global", "user.name", "PIA Servers CI");
+            runCommand("git", "config", "--global", "user.email", "fascinated-helper@fascinated.cc");
+            runCommand("git", "config", "--global", "user.name", "Fascinated's Helper");
             for (Path file : files) {
                 runCommand("git", "add", file.toAbsolutePath().toString());
             }
             runCommand("git", "commit", "-m", message);
-            runCommand("git", "push", "https://pia-servers-ci:%s@git.fascinated.cc/Fascinated/PIA-Servers".formatted(System.getenv("AUTH_TOKEN")));
+            runCommand("git", "push", "https://fascinated-helper:%s@git.fascinated.cc/Fascinated/PIA-Servers".formatted(System.getenv("AUTH_TOKEN")));
         }
     }
 

src/main/java/cc/fascinated/piaservers/model/PiaServerToken.java

@@ -1,17 +0,0 @@
-package cc.fascinated.piaservers.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-
-@AllArgsConstructor @Getter
-public class PiaServerToken {
-    /**
-     * The ip for this server.
-     */
-    private final String ip;
-
-    /**
-     * The region this server is in.
-     */
-    private final String region;
-}

src/main/java/cc/fascinated/piaservers/pia/PiaManager.java

@@ -3,7 +3,6 @@ package cc.fascinated.piaservers.pia;
 import cc.fascinated.piaservers.Main;
 import cc.fascinated.piaservers.common.GitUtils;
 import cc.fascinated.piaservers.model.PiaServer;
-import cc.fascinated.piaservers.model.PiaServerToken;
 import cc.fascinated.piaservers.readme.ReadMeManager;
 import com.google.gson.reflect.TypeToken;
 import lombok.SneakyThrows;
@@ -12,6 +11,7 @@ import org.xbill.DNS.Record;
 import org.xbill.DNS.*;
 
 import java.io.File;
+import java.io.IOException;
 import java.net.URI;
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
@@ -32,7 +32,7 @@ public class PiaManager {
     public PiaManager() {
         File serversFile = new File("servers.json");
         if (!serversFile.exists()) {
-            System.out.println("servers.json does not exist, creating...");
+            System.out.println("The servers file doesn't exist, creating it...");
             serversFile.createNewFile();
         }
         // Load the serversFile from the file
@@ -40,18 +40,21 @@ public class PiaManager {
         if (SERVERS == null) {
             SERVERS = new HashSet<>();
         }
-        System.out.println("Loaded " + SERVERS.size() + " servers from the file");
+        System.out.printf("Loaded %s servers from the file%n", SERVERS.size());
+
+        // Set the DNS resolver to Cloudflare
+        Lookup.setDefaultResolver(new SimpleResolver("1.1.1.1"));
 
         GitUtils.cloneRepo(); // Clone the repository
 
-        // Update the servers every 2 minutes
+        // Update the servers every 5 minutes
         new Timer().scheduleAtFixedRate(new TimerTask() {
             @Override
             public void run() {
                 updateServers(serversFile); // Update the servers
                 README_PATH = ReadMeManager.updateReadme(); // Update the README.md
             }
-        }, 0, TimeUnit.MINUTES.toMillis(2));
+        }, 0, TimeUnit.MINUTES.toMillis(5));
 
         // Commit the files every hour
         new Timer().scheduleAtFixedRate(new TimerTask() {
@@ -64,100 +67,76 @@ public class PiaManager {
 
     @SneakyThrows
     public static void updateServers(File serversFile) {
-        List<PiaServerToken> piaDomain = getPiaDomains();
-        System.out.println("Found " + piaDomain.size() + " pia domains");
+        List<PiaServer> servers = getPiaServers();
 
-        List<PiaServer> toRemove = new ArrayList<>();
-
-        System.out.println("Removing old servers...");
-        // Get the servers that need to be removed
-        for (PiaServer server : SERVERS) {
-            if (server.getLastSeen().getTime() < System.currentTimeMillis() - REMOVAL_THRESHOLD) {
-                toRemove.add(server);
-            }
-        }
-        toRemove.forEach(SERVERS::remove); // Remove the servers
-        System.out.printf("Removed %s old servers\n", toRemove.size());
-
-        int newServers = 0;
+        // Remove the servers that haven't been active in 2 weeks
+        int before = SERVERS.size();
+        SERVERS.removeIf(server -> System.currentTimeMillis() - server.getLastSeen().getTime() > REMOVAL_THRESHOLD);
+        System.out.printf("Removed %s servers that haven't been active in 2 weeks%n", before - SERVERS.size());
 
         // Add the new servers to the list
-        for (PiaServerToken serverToken : piaDomain) {
-            boolean newServer = SERVERS.stream().noneMatch(server -> server.getIp().equals(serverToken.getIp()));
+        int newServers = 0;
+        for (PiaServer piaServer : servers) {
+            boolean newServer = SERVERS.stream().noneMatch(server -> server.getIp().equals(piaServer.getIp()));
             if (newServer) {
                 newServers++;
             }
 
             // Add the server to the list
-            SERVERS.add(new PiaServer(serverToken.getIp(), serverToken.getRegion(), new Date()));
+            SERVERS.add(piaServer);
         }
 
         // Save the servers to the file
         Files.writeString(serversFile.toPath(), Main.GSON.toJson(SERVERS));
-        System.out.printf("Wrote %s new servers to the file\n", newServers);
+        System.out.printf("Wrote %s servers to the file (+%s new)%n", SERVERS.size(), newServers);
     }
 
     @SneakyThrows
-    private static List<PiaServerToken> getPiaDomains() {
+    private static List<PiaServer> getPiaServers() {
+        long start = System.currentTimeMillis();
         HttpRequest request = HttpRequest.newBuilder()
                 .uri(URI.create(PIA_OPENVPN_CONFIGS_URL))
                 .GET()
                 .build();
-        // Send the request and get the response
         HttpResponse<Path> response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofFile(Files.createTempFile("openvpn", ".zip")));
         if (response.statusCode() != 200) {
-            System.out.println("Failed to get the PIA OpenVPN configs, status code: " + response.statusCode());
-            System.exit(1);
+            throw new IOException("Failed to get the PIA OpenVPN configs, status code: " + response.statusCode());
         }
+        System.out.printf("Downloaded the OpenVPN configs in %sms%n", System.currentTimeMillis() - start);
         Path downloadedFile = response.body();
         File tempDir = Files.createTempDirectory("openvpn").toFile();
         ZipUnArchiver unArchiver = new ZipUnArchiver();
-
-        // Extract the downloaded file
         unArchiver.setSourceFile(downloadedFile.toFile());
         unArchiver.setDestDirectory(tempDir);
         unArchiver.extract();
 
-        // Get the extracted files
         File[] files = tempDir.listFiles();
         if (files == null || files.length == 0) {
-            System.out.println("Failed to extract the OpenVPN configs");
-            System.exit(1);
+            throw new IOException("Failed to extract the OpenVPN configs");
         }
 
-        // Set the DNS resolver to Cloudflare
-        Lookup.setDefaultResolver(new SimpleResolver("1.1.1.1"));
-
-        // Search for the server domains
-        List<PiaServerToken> domains = new ArrayList<>();
+        List<PiaServer> servers = new ArrayList<>();
         for (File file : files) {
-            if (file.isDirectory()) {
+            if (file.isDirectory() || !file.getName().endsWith(".ovpn")) {
                 continue;
             }
-            if (!file.getName().endsWith(".ovpn")) {
-                continue;
-            }
-            // Read the file and get the server domain
             List<String> lines = Files.readAllLines(file.toPath());
             for (String line : lines) {
                 if (line.startsWith("remote ")) {
                     String[] parts = line.split(" ");
                     String hostname = parts[1];
                     String region = file.getName().split("\\.")[0];
-
                     Record[] records = new Lookup(hostname, Type.A).run();
-                    if (records == null) {
-                        continue;
-                    }
-                    for (Record record : records) {
-                        ARecord aRecord = (ARecord) record;
-                        domains.add(new PiaServerToken(aRecord.getAddress().getHostAddress(), region));
+                    if (records != null) {
+                        for (Record record : records) {
+                            ARecord aRecord = (ARecord) record;
+                            servers.add(new PiaServer(aRecord.getAddress().getHostAddress(), region, new Date()));
+                        }
                     }
                     break;
                 }
             }
         }
-
-        return domains;
+        return servers;
     }
 }

src/main/java/cc/fascinated/piaservers/readme/ReadMeManager.java

@@ -19,6 +19,7 @@ public class ReadMeManager {
 
     @SneakyThrows
     public static Path updateReadme() {
+        System.out.println("Updating README.md");
         InputStream readmeStream = Main.class.getResourceAsStream("/README.md");
         if (readmeStream == null) {
             System.out.println("Failed to find README.md");
@@ -49,6 +50,7 @@ public class ReadMeManager {
                 .reduce((a, b) -> a + "\n" + b).orElse("")); // Reduce the entries to a single string
 
         Files.write(readmeFile.toPath(), contents.getBytes());
+        System.out.println("Finished updating README.md");
         return readmeFile.toPath();
     }
 }

src/main/resources/README.md

@@ -1,8 +1,11 @@
 # PIA Server List
 
-![Servers](https://img.shields.io/badge/servers-{server_count}-blue) ![Regions](https://img.shields.io/badge/regions-{region_count}-blue) ![Last update](https://img.shields.io/badge/last_updated-{last_update}-blue)
+![Servers](https://img.shields.io/badge/servers-{server_count}-blue)
+![Regions](https://img.shields.io/badge/regions-{region_count}-blue)
+![Last update](https://img.shields.io/badge/last_updated-{last_update}-blue)
 
-This is a list of the OpenVPN servers provided by Private Internet Access (PIA). The list is updated hourly.
+This is a list of the OpenVPN servers provided by Private Internet Access (PIA). The list is updated hourly. </br>
+Servers that have not been active in the last 2 weeks will be automatically removed.
 
 ## Servers
 | Region               | Servers |