Scheduled update

Reviewed-on: #1

Dockerfile

@@ -25,5 +25,8 @@ WORKDIR /home/container
 # Copy the built jar file from the builder stage
 COPY --from=builder /home/container/target/PIA-Servers.jar .
 
+# Copy the old servers.json file
+COPY --from=builder /home/container/servers.json .
+
 # Run the jar file
 CMD java -jar PIA-Servers.jar -Djava.awt.headless=true

README.md

@@ -1,106 +1,109 @@
 # PIA Server List
 
-![Servers](https://img.shields.io/badge/servers-13,907-blue) ![Regions](https://img.shields.io/badge/regions-97-blue) ![Last update](https://img.shields.io/badge/last_updated-Mon_Apr_29_05:33:33_GMT_2024-blue)
+![Servers](https://img.shields.io/badge/servers-18,240-blue)
+![Regions](https://img.shields.io/badge/regions-97-blue)
+![Last update](https://img.shields.io/badge/last_updated-Wed_Jul_31_17:54:31_UTC_2024-blue)
 
-This is a list of the OpenVPN servers provided by Private Internet Access (PIA). The list is updated hourly.
+This is a list of the OpenVPN servers provided by Private Internet Access (PIA). The list is updated hourly. </br>
+Servers that have not been active in the last 2 weeks will be automatically removed.
 
 ## Servers
 | Region               | Servers |
 |----------------------|---------|
-| us_florida | 148 |
-| ca_vancouver | 148 |
-| us_seattle | 148 |
-| philippines | 148 |
-| us_east | 148 |
-| de_berlin | 148 |
-| uk_manchester | 148 |
-| switzerland | 148 |
-| ukraine | 148 |
-| spain | 148 |
-| panama | 148 |
-| us_california | 148 |
-| singapore | 148 |
-| mexico | 148 |
-| us_new_york | 148 |
-| uk_london | 148 |
-| us_chicago | 148 |
-| us_washington_dc | 148 |
-| netherlands | 148 |
-| us_texas | 148 |
-| de_frankfurt | 148 |
-| us_silicon_valley | 148 |
-| au_melbourne | 148 |
-| new_zealand | 148 |
-| us_houston | 148 |
-| ca_montreal | 148 |
-| israel | 148 |
-| us_atlanta | 148 |
-| ireland | 148 |
-| greece | 147 |
-| france | 147 |
-| brazil | 147 |
-| romania | 147 |
-| japan | 146 |
-| poland | 146 |
-| moldova | 146 |
-| estonia | 146 |
-| sweden | 146 |
-| us_las_vegas | 146 |
-| us_west | 146 |
-| bahamas | 146 |
-| ca_toronto | 146 |
-| taiwan | 146 |
-| czech_republic | 146 |
-| uk_southampton | 146 |
-| denmark | 146 |
-| ca_ontario | 146 |
-| qatar | 146 |
-| luxembourg | 145 |
-| italy | 145 |
-| belgium | 145 |
-| united_arab_emirates | 144 |
-| austria | 144 |
-| lithuania | 144 |
-| norway | 144 |
-| au_perth | 144 |
-| georgia | 144 |
-| argentina | 144 |
-| au_sydney | 144 |
-| kazakhstan | 144 |
-| vietnam | 143 |
-| greenland | 143 |
-| isle_of_man | 143 |
-| south_africa | 143 |
-| venezuela | 142 |
-| nigeria | 142 |
-| liechtenstein | 142 |
-| slovakia | 142 |
-| latvia | 142 |
-| malta | 142 |
-| saudi_arabia | 141 |
-| cyprus | 141 |
-| bangladesh | 141 |
-| algeria | 140 |
-| hungary | 140 |
-| finland | 140 |
-| india | 140 |
-| egypt | 140 |
-| macedonia | 140 |
-| china | 140 |
-| monaco | 139 |
-| albania | 138 |
-| andorra | 138 |
-| cambodia | 138 |
-| iceland | 138 |
-| us_denver | 137 |
-| montenegro | 137 |
-| armenia | 136 |
-| turkey | 136 |
-| macao | 135 |
-| bulgaria | 132 |
-| morocco | 132 |
-| portugal | 130 |
-| serbia | 130 |
-| mongolia | 128 |
-| hong_kong | 128 |
-| sri_lanka | 127 |
+| ca_montreal | 816 |
+| netherlands | 811 |
+| us_texas | 708 |
+| us_chicago | 702 |
+| ca_toronto | 643 |
+| us_silicon_valley | 601 |
+| us_washington_dc | 589 |
+| us_atlanta | 565 |
+| us_west | 559 |
+| ca_vancouver | 547 |
+| us_las_vegas | 542 |
+| us_new_york | 541 |
+| us_east | 536 |
+| uk_london | 533 |
+| us_california | 526 |
+| us_florida | 496 |
+| us_seattle | 496 |
+| ca_ontario | 430 |
+| au_melbourne | 426 |
+| switzerland | 420 |
+| de_frankfurt | 417 |
+| us_houston | 384 |
+| au_sydney | 355 |
+| uk_southampton | 318 |
+| japan | 308 |
+| de_berlin | 261 |
+| us_denver | 231 |
+| uk_manchester | 224 |
+| singapore | 218 |
+| france | 202 |
+| sweden | 192 |
+| au_perth | 183 |
+| new_zealand | 183 |
+| mexico | 139 |
+| ireland | 132 |
+| romania | 120 |
+| estonia | 117 |
+| norway | 112 |
+| belgium | 102 |
+| israel | 100 |
+| spain | 99 |
+| denmark | 96 |
+| czech_republic | 94 |
+| italy | 87 |
+| finland | 84 |
+| bahamas | 78 |
+| argentina | 75 |
+| austria | 71 |
+| panama | 71 |
+| brazil | 69 |
+| poland | 64 |
+| luxembourg | 62 |
+| latvia | 60 |
+| ukraine | 59 |
+| india | 59 |
+| lithuania | 56 |
+| greenland | 56 |
+| turkey | 52 |
+| slovakia | 52 |
+| andorra | 48 |
+| vietnam | 46 |
+| taiwan | 46 |
+| venezuela | 45 |
+| greece | 45 |
+| portugal | 43 |
+| united_arab_emirates | 38 |
+| china | 38 |
+| malta | 37 |
+| albania | 36 |
+| philippines | 36 |
+| algeria | 36 |
+| qatar | 33 |
+| bangladesh | 31 |
+| hungary | 30 |
+| liechtenstein | 29 |
+| isle_of_man | 29 |
+| saudi_arabia | 28 |
+| bulgaria | 28 |
+| south_africa | 28 |
+| macao | 26 |
+| montenegro | 26 |
+| nigeria | 25 |
+| moldova | 24 |
+| cambodia | 24 |
+| cyprus | 24 |
+| morocco | 24 |
+| egypt | 24 |
+| iceland | 24 |
+| kazakhstan | 23 |
+| macedonia | 23 |
+| monaco | 22 |
+| georgia | 22 |
+| hong_kong | 20 |
+| serbia | 15 |
+| armenia | 12 |
+| mongolia | 12 |
+| sri_lanka | 11 |

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "local>Fascinated/renovate-config"
+  ]
+}

src/main/java/cc/fascinated/piaservers/common/Config.java

@@ -0,0 +1,13 @@
+package cc.fascinated.piaservers.common;
+
+public class Config {
+
+    /**
+     * Are we in production?
+     *
+     * @return If we are in production
+     */
+    public static boolean isProduction() {
+        return System.getenv().containsKey("ENVIRONMENT") && System.getenv("ENVIRONMENT").equals("production");
+    }
+}

src/main/java/cc/fascinated/piaservers/common/GitUtils.java

@@ -12,26 +12,27 @@ public class GitUtils {
      */
     public static void commitFiles(String message, Path... files) {
         System.out.println("Committing files");
-        if (System.getenv("ENVIRONMENT").equals("production")) {
-            runCommand("git", "config", "--global", "user.email", "liam+pia-servers-ci@fascinated.cc");
-            runCommand("git", "config", "--global", "user.name", "PIA Servers CI");
+        if (Config.isProduction()) {
+            runCommand("git", "config", "--global", "user.email", "fascinated-helper@fascinated.cc");
+            runCommand("git", "config", "--global", "user.name", "Fascinated's Helper");
+            for (Path file : files) {
+                runCommand("git", "add", file.toAbsolutePath().toString());
+            }
+            runCommand("git", "commit", "-m", message);
+            runCommand("git", "push", "https://fascinated-helper:%s@git.fascinated.cc/Fascinated/PIA-Servers".formatted(System.getenv("AUTH_TOKEN")));
         }
-        for (Path file : files) {
-            runCommand("git", "add", file.toAbsolutePath().toString());
-        }
-        runCommand("git", "commit", "-m", message);
-        runCommand("git", "push", "https://pia-servers-ci:%s@git.fascinated.cc/Fascinated/PIA-Servers".formatted(System.getenv("AUTH_TOKEN")));
     }
 
     /**
      * Clone the repository
      */
     public static void cloneRepo() {
-        if (System.getenv("ENVIRONMENT").equals("production")) {
+        if (Config.isProduction()) {
             System.out.println("Cloning repository");
             runCommand("git", "clone", "https://git.fascinated.cc/Fascinated/PIA-Servers.git");
             runCommand("mv", "PIA-Servers/.git", ".");
             runCommand("rm", "-rf", "PIA-Servers");
+            runCommand("git", "pull"); // Pull the latest changes
         }
     }
 

src/main/java/cc/fascinated/piaservers/model/PiaServerToken.java

@@ -1,17 +0,0 @@
-package cc.fascinated.piaservers.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-
-@AllArgsConstructor @Getter
-public class PiaServerToken {
-    /**
-     * The hostname for this server.
-     */
-    private final String hostname;
-
-    /**
-     * The region this server is in.
-     */
-    private final String region;
-}

src/main/java/cc/fascinated/piaservers/pia/PiaManager.java

@@ -3,7 +3,6 @@ package cc.fascinated.piaservers.pia;
 import cc.fascinated.piaservers.Main;
 import cc.fascinated.piaservers.common.GitUtils;
 import cc.fascinated.piaservers.model.PiaServer;
-import cc.fascinated.piaservers.model.PiaServerToken;
 import cc.fascinated.piaservers.readme.ReadMeManager;
 import com.google.gson.reflect.TypeToken;
 import lombok.SneakyThrows;
@@ -12,7 +11,7 @@ import org.xbill.DNS.Record;
 import org.xbill.DNS.*;
 
 import java.io.File;
-import java.net.InetAddress;
+import java.io.IOException;
 import java.net.URI;
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
@@ -27,125 +26,117 @@ public class PiaManager {
     private static final String PIA_OPENVPN_CONFIGS_URL = "https://www.privateinternetaccess.com/openvpn/openvpn.zip";
     private static final long REMOVAL_THRESHOLD = TimeUnit.DAYS.toMicros(14); // 2 weeks
     public static Set<PiaServer> SERVERS = new HashSet<>();
+    private static Path README_PATH;
 
     @SneakyThrows
     public PiaManager() {
         File serversFile = new File("servers.json");
         if (!serversFile.exists()) {
-            System.out.println("serversFile.json does not exist, creating...");
+            System.out.println("The servers file doesn't exist, creating it...");
             serversFile.createNewFile();
         }
         // Load the serversFile from the file
-        SERVERS = Main.GSON.fromJson(Files.readString(serversFile.toPath()), new TypeToken<List<PiaServer>>() {}.getType());
+        SERVERS = Main.GSON.fromJson(Files.readString(serversFile.toPath()), new TypeToken<Set<PiaServer>>() {}.getType());
         if (SERVERS == null) {
             SERVERS = new HashSet<>();
         }
-
-        GitUtils.cloneRepo(); // Clone the repository
-
-        new Timer().scheduleAtFixedRate(new TimerTask() {
-            @Override
-            public void run() {
-                updateServers(serversFile); // Update the servers
-                Path readmePath = ReadMeManager.updateReadme(); // Update the README.md
-
-                // Commit the changes to the git repository
-                GitUtils.commitFiles("Scheduled update", serversFile.toPath(), readmePath);
-            }
-        }, 0, TimeUnit.MINUTES.toMillis(5));
-    }
-
-    @SneakyThrows
-    public static void updateServers(File serversFile) {
-        List<PiaServerToken> piaDomain = getPiaDomains();
-        System.out.println("Found " + piaDomain.size() + " pia domains");
-
-        List<PiaServer> toRemove = new ArrayList<>();
-
-        System.out.println("Removing old servers...");
-        // Get the servers that need to be removed
-        for (PiaServer server : SERVERS) {
-            if (server.getLastSeen().getTime() < System.currentTimeMillis() - REMOVAL_THRESHOLD) {
-                toRemove.add(server);
-            }
-        }
-        SERVERS.removeAll(toRemove); // Remove the servers
-        System.out.printf("Removed %s old servers\n", toRemove.size());
-
-        // Add the new servers to the list
-        for (PiaServerToken serverToken : piaDomain) {
-            InetAddress address = InetAddress.getByName(serverToken.getHostname());
-
-            // Add the server to the list
-            SERVERS.add(new PiaServer(address.getHostAddress(), serverToken.getRegion(), new Date()));
-        }
-
-        // Save the servers to the file
-        Files.writeString(serversFile.toPath(), Main.GSON.toJson(SERVERS));
-        System.out.printf("Wrote %s servers to the file\n", SERVERS.size());
-    }
-
-    @SneakyThrows
-    private static List<PiaServerToken> getPiaDomains() {
-        HttpRequest request = HttpRequest.newBuilder()
-                .uri(URI.create(PIA_OPENVPN_CONFIGS_URL))
-                .GET()
-                .build();
-        // Send the request and get the response
-        HttpResponse<Path> response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofFile(Files.createTempFile("openvpn", ".zip")));
-        if (response.statusCode() != 200) {
-            System.out.println("Failed to get the PIA OpenVPN configs, status code: " + response.statusCode());
-            System.exit(1);
-        }
-        Path downloadedFile = response.body();
-        File tempDir = Files.createTempDirectory("openvpn").toFile();
-        ZipUnArchiver unArchiver = new ZipUnArchiver();
-
-        // Extract the downloaded file
-        unArchiver.setSourceFile(downloadedFile.toFile());
-        unArchiver.setDestDirectory(tempDir);
-        unArchiver.extract();
-
-        // Get the extracted files
-        File[] files = tempDir.listFiles();
-        if (files == null || files.length == 0) {
-            System.out.println("Failed to extract the OpenVPN configs");
-            System.exit(1);
-        }
+        System.out.printf("Loaded %s servers from the file%n", SERVERS.size());
 
         // Set the DNS resolver to Cloudflare
         Lookup.setDefaultResolver(new SimpleResolver("1.1.1.1"));
 
-        // Search for the server domains
-        List<PiaServerToken> domains = new ArrayList<>();
+        GitUtils.cloneRepo(); // Clone the repository
+
+        // Update the servers every 5 minutes
+        new Timer().scheduleAtFixedRate(new TimerTask() {
+            @Override
+            public void run() {
+                updateServers(serversFile); // Update the servers
+                README_PATH = ReadMeManager.updateReadme(); // Update the README.md
+            }
+        }, 0, TimeUnit.MINUTES.toMillis(5));
+
+        // Commit the files every hour
+        new Timer().scheduleAtFixedRate(new TimerTask() {
+            @Override
+            public void run() {
+                GitUtils.commitFiles("Scheduled update", serversFile.toPath(), README_PATH); // Commit the files
+            }
+        }, TimeUnit.MINUTES.toMillis(3), TimeUnit.HOURS.toMillis(1));
+    }
+
+    @SneakyThrows
+    public static void updateServers(File serversFile) {
+        List<PiaServer> servers = getPiaServers();
+
+        // Remove the servers that haven't been active in 2 weeks
+        int before = SERVERS.size();
+        SERVERS.removeIf(server -> System.currentTimeMillis() - server.getLastSeen().getTime() > REMOVAL_THRESHOLD);
+        System.out.printf("Removed %s servers that haven't been active in 2 weeks%n", before - SERVERS.size());
+
+        // Add the new servers to the list
+        int newServers = 0;
+        for (PiaServer piaServer : servers) {
+            boolean newServer = SERVERS.stream().noneMatch(server -> server.getIp().equals(piaServer.getIp()));
+            if (newServer) {
+                newServers++;
+            }
+
+            // Add the server to the list
+            SERVERS.add(piaServer);
+        }
+
+        // Save the servers to the file
+        Files.writeString(serversFile.toPath(), Main.GSON.toJson(SERVERS));
+        System.out.printf("Wrote %s servers to the file (+%s new)%n", SERVERS.size(), newServers);
+    }
+
+    @SneakyThrows
+    private static List<PiaServer> getPiaServers() {
+        long start = System.currentTimeMillis();
+        HttpRequest request = HttpRequest.newBuilder()
+                .uri(URI.create(PIA_OPENVPN_CONFIGS_URL))
+                .GET()
+                .build();
+        HttpResponse<Path> response = HTTP_CLIENT.send(request, HttpResponse.BodyHandlers.ofFile(Files.createTempFile("openvpn", ".zip")));
+        if (response.statusCode() != 200) {
+            throw new IOException("Failed to get the PIA OpenVPN configs, status code: " + response.statusCode());
+        }
+        System.out.printf("Downloaded the OpenVPN configs in %sms%n", System.currentTimeMillis() - start);
+        Path downloadedFile = response.body();
+        File tempDir = Files.createTempDirectory("openvpn").toFile();
+        ZipUnArchiver unArchiver = new ZipUnArchiver();
+        unArchiver.setSourceFile(downloadedFile.toFile());
+        unArchiver.setDestDirectory(tempDir);
+        unArchiver.extract();
+
+        File[] files = tempDir.listFiles();
+        if (files == null || files.length == 0) {
+            throw new IOException("Failed to extract the OpenVPN configs");
+        }
+
+        List<PiaServer> servers = new ArrayList<>();
         for (File file : files) {
-            if (file.isDirectory()) {
+            if (file.isDirectory() || !file.getName().endsWith(".ovpn")) {
                 continue;
             }
-            if (!file.getName().endsWith(".ovpn")) {
-                continue;
-            }
-            // Read the file and get the server domain
             List<String> lines = Files.readAllLines(file.toPath());
             for (String line : lines) {
                 if (line.startsWith("remote ")) {
                     String[] parts = line.split(" ");
                     String hostname = parts[1];
                     String region = file.getName().split("\\.")[0];
-
                     Record[] records = new Lookup(hostname, Type.A).run();
-                    if (records == null) {
-                        continue;
-                    }
-                    for (Record record : records) {
-                        ARecord aRecord = (ARecord) record;
-                        domains.add(new PiaServerToken(aRecord.getAddress().getHostAddress(), region));
+                    if (records != null) {
+                        for (Record record : records) {
+                            ARecord aRecord = (ARecord) record;
+                            servers.add(new PiaServer(aRecord.getAddress().getHostAddress(), region, new Date()));
+                        }
                     }
                     break;
                 }
             }
         }
-
-        return domains;
+        return servers;
     }
 }

src/main/java/cc/fascinated/piaservers/readme/ReadMeManager.java

@@ -19,6 +19,7 @@ public class ReadMeManager {
 
     @SneakyThrows
     public static Path updateReadme() {
+        System.out.println("Updating README.md");
         InputStream readmeStream = Main.class.getResourceAsStream("/README.md");
         if (readmeStream == null) {
             System.out.println("Failed to find README.md");
@@ -49,6 +50,7 @@ public class ReadMeManager {
                 .reduce((a, b) -> a + "\n" + b).orElse("")); // Reduce the entries to a single string
 
         Files.write(readmeFile.toPath(), contents.getBytes());
+        System.out.println("Finished updating README.md");
         return readmeFile.toPath();
     }
 }

src/main/resources/README.md

@@ -1,8 +1,11 @@
 # PIA Server List
 
-![Servers](https://img.shields.io/badge/servers-{server_count}-blue) ![Regions](https://img.shields.io/badge/regions-{region_count}-blue) ![Last update](https://img.shields.io/badge/last_updated-{last_update}-blue)
+![Servers](https://img.shields.io/badge/servers-{server_count}-blue)
+![Regions](https://img.shields.io/badge/regions-{region_count}-blue)
+![Last update](https://img.shields.io/badge/last_updated-{last_update}-blue)
 
-This is a list of the OpenVPN servers provided by Private Internet Access (PIA). The list is updated hourly.
+This is a list of the OpenVPN servers provided by Private Internet Access (PIA). The list is updated hourly. </br>
+Servers that have not been active in the last 2 weeks will be automatically removed.
 
 ## Servers
 | Region               | Servers |