Merge pull request #447 from codeshard/codeshard/poetry

feat: added improvements

.dockerignore

@@ -1,4 +1,7 @@
 venv/
 .env*
 db.sqlite3
-.github
+.github/
+.git/
+.idea/
+.venv/

proxylist/admin.py → app/shadowmere/apps/proxylist/admin.py

@@ -1,13 +1,12 @@
 import humanfriendly
+from apps.proxylist.models import Proxy, Subscription, TaskLog
 from django.contrib import admin
 from django.contrib.auth.models import Group
 from django.db import IntegrityError
 from import_export import resources
 from import_export.admin import ImportExportModelAdmin
 from rangefilter.filters import DateRangeFilter
-
-from proxylist.models import Proxy, Subscription, TaskLog
-from proxylist.proxy import update_proxy_status
+from utils.proxy import update_proxy_status
 
 
 class ProxyResource(resources.ModelResource):
@@ -17,7 +16,8 @@ class Meta:
 
 @admin.register(Proxy)
 class ProxyAdmin(ImportExportModelAdmin):
-    def update_status(modeladmin, request, queryset):
+    @staticmethod
+    def update_status(queryset):
         for proxy in queryset:
             update_proxy_status(proxy)
             try:
@@ -26,7 +26,8 @@ def update_status(modeladmin, request, queryset):
                 # This means the proxy is either a duplicate or no longer valid
                 proxy.delete()
 
-    def quality(self, obj):
+    @staticmethod
+    def quality(obj):
         if obj.times_checked > 0:
             return obj.times_check_succeeded * 100 / obj.times_checked
         else:

app/shadowmere/apps/proxylist/apps.py

@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+
+
+class ProxylistConfig(AppConfig):
+    name = "apps.proxylist"
+
+    def ready(self) -> None:
+        import apps.proxylist.signals  # noqa

proxylist/management/commands/clear_stats.py → app/shadowmere/apps/proxylist/management/commands/clear_stats.py

@@ -1,7 +1,6 @@
+from apps.proxylist.models import Proxy
 from django.core.management.base import BaseCommand
 
-from proxylist.models import Proxy
-
 
 class Command(BaseCommand):
     help = "Clear all quality statistics"

proxylist/management/commands/poll_subscriptions.py → app/shadowmere/apps/proxylist/management/commands/poll_subscriptions.py

@@ -1,7 +1,6 @@
+from apps.proxylist.tasks import poll_subscriptions
 from django.core.management.base import BaseCommand
 
-from proxylist.tasks import poll_subscriptions
-
 
 class Command(BaseCommand):
     help = "Poll all subscriptions looking for new keys"

proxylist/management/commands/reduce_checks_by.py → app/shadowmere/apps/proxylist/management/commands/reduce_checks_by.py

@@ -1,7 +1,6 @@
+from apps.proxylist.models import Proxy
 from django.core.management.base import BaseCommand
 
-from proxylist.models import Proxy
-
 
 class Command(BaseCommand):
     help = "Clear all quality statistics"

proxylist/management/commands/remove_low_quality.py → app/shadowmere/apps/proxylist/management/commands/remove_low_quality.py

@@ -1,8 +1,7 @@
-from django.core.management.base import BaseCommand
-
-from proxylist.tasks import (
+from apps.proxylist.tasks import (
     remove_low_quality_proxies,
 )
+from django.core.management.base import BaseCommand
 
 
 class Command(BaseCommand):

proxylist/management/commands/update_status.py → app/shadowmere/apps/proxylist/management/commands/update_status.py

@@ -1,7 +1,6 @@
+from apps.proxylist.tasks import update_status
 from django.core.management.base import BaseCommand
 
-from proxylist.tasks import update_status
-
 
 class Command(BaseCommand):
     help = "Update the status of the proxy list"

proxylist/metrics.py → app/shadowmere/apps/proxylist/metrics.py

@@ -1,8 +1,7 @@
-from prometheus_client.core import GaugeMetricFamily, REGISTRY
+from apps.proxylist.models import Proxy
+from prometheus_client.core import REGISTRY, GaugeMetricFamily
 from prometheus_client.registry import Collector
 
-from proxylist.models import Proxy
-
 
 class CustomCollector(Collector):
     def collect(self):

proxylist/migrations/0002_proxy_last_active.py → app/shadowmere/apps/proxylist/migrations/0002_proxy_last_active.py

@@ -5,7 +5,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("proxylist", "0001_initial"),
     ]

proxylist/migrations/0003_alter_proxy_url.py → app/shadowmere/apps/proxylist/migrations/0003_alter_proxy_url.py

@@ -1,11 +1,10 @@
 # Generated by Django 3.2.11 on 2022-02-09 14:26
 
 from django.db import migrations, models
-import proxylist.models
+import apps.proxylist.models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("proxylist", "0002_proxy_last_active"),
     ]
@@ -17,7 +16,6 @@ class Migration(migrations.Migration):
             field=models.CharField(
                 max_length=1024,
                 unique=True,
-                validators=[proxylist.models.validate_not_existing],
             ),
         ),
     ]

proxylist/migrations/0004_proxy_times_check_failed_proxy_times_checked.py → app/shadowmere/apps/proxylist/migrations/0004_proxy_times_check_failed_proxy_times_checked.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("proxylist", "0003_alter_proxy_url"),
     ]

proxylist/migrations/0005_remove_proxy_times_check_failed.py → app/shadowmere/apps/proxylist/migrations/0005_remove_proxy_times_check_failed.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("proxylist", "0004_proxy_times_check_failed_proxy_times_checked"),
     ]

proxylist/migrations/0006_proxy_times_check_succeeded.py → app/shadowmere/apps/proxylist/migrations/0006_proxy_times_check_succeeded.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("proxylist", "0005_remove_proxy_times_check_failed"),
     ]

proxylist/migrations/0007_proxy_ip_address_proxy_location_country_code.py → app/shadowmere/apps/proxylist/migrations/0007_proxy_ip_address_proxy_location_country_code.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("proxylist", "0006_proxy_times_check_succeeded"),
     ]

app/shadowmere/apps/proxylist/migrations/0008_proxy_location_country_alter_proxy_url.py

@@ -0,0 +1,30 @@
+# Generated by Django 4.0.4 on 2022-06-06 12:09
+
+from django.db import migrations, models
+from utils import validators
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("proxylist", "0007_proxy_ip_address_proxy_location_country_code"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="proxy",
+            name="location_country",
+            field=models.CharField(default="", max_length=50),
+        ),
+        migrations.AlterField(
+            model_name="proxy",
+            name="url",
+            field=models.CharField(
+                max_length=1024,
+                unique=True,
+                validators=[
+                    validators.validate_sip002,
+                    validators.validate_proxy_can_connect,
+                ],
+            ),
+        ),
+    ]

proxylist/migrations/0009_proxy_port.py → app/shadowmere/apps/proxylist/migrations/0009_proxy_port.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("proxylist", "0008_proxy_location_country_alter_proxy_url"),
     ]

app/shadowmere/apps/proxylist/migrations/0010_subscription_alter_proxy_url.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.7 on 2023-03-04 14:37
+
+from django.db import migrations, models
+from utils import validators
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("proxylist", "0009_proxy_port"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Subscription",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("url", models.URLField(unique=True)),
+                ("kind", models.CharField(choices=[("1", "plain"), ("2", "base64")], default="1", max_length=10)),
+            ],
+        ),
+        migrations.AlterField(
+            model_name="proxy",
+            name="url",
+            field=models.CharField(max_length=1024, unique=True, validators=[validators.proxy_validator]),
+        ),
+    ]

app/shadowmere/apps/proxylist/migrations/0011_alter_subscription_kind.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.1.7 on 2023-03-04 14:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("proxylist", "0010_subscription_alter_proxy_url"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="subscription",
+            name="kind",
+            field=models.CharField(choices=[("PLAIN", "plain"), ("BASE64", "base64")], default="PLAIN", max_length=10),
+        ),
+    ]

app/shadowmere/apps/proxylist/migrations/0018_alter_proxy_ip_address_alter_proxy_is_active_and_more.py

@@ -0,0 +1,43 @@
+# Generated by Django 5.0.6 on 2024-06-08 18:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('proxylist', '0017_tasklog_details'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='proxy',
+            name='ip_address',
+            field=models.CharField(blank=True, db_index=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='proxy',
+            name='is_active',
+            field=models.BooleanField(db_index=True, default=False),
+        ),
+        migrations.AlterField(
+            model_name='proxy',
+            name='location',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='proxy',
+            name='location_country',
+            field=models.CharField(blank=True, db_index=True, max_length=50, null=True),
+        ),
+        migrations.AlterField(
+            model_name='proxy',
+            name='location_country_code',
+            field=models.CharField(blank=True, db_index=True, max_length=3, null=True),
+        ),
+        migrations.AlterField(
+            model_name='proxy',
+            name='port',
+            field=models.IntegerField(db_index=True, default=0),
+        ),
+    ]

app/shadowmere/apps/proxylist/models.py

@@ -0,0 +1,53 @@
+from django.db import models
+from django.utils.timezone import now
+from django_prometheus.models import ExportModelOperationsMixin
+from utils.validators import proxy_validator
+
+
+class Proxy(ExportModelOperationsMixin("proxy"), models.Model):
+    url = models.CharField(
+        max_length=1024,
+        unique=True,
+        validators=[
+            proxy_validator,
+        ],
+    )
+    location = models.CharField(max_length=100, blank=True, null=True)
+    location_country_code = models.CharField(max_length=3, blank=True, null=True, db_index=True)
+    location_country = models.CharField(max_length=50, blank=True, null=True, db_index=True)
+    ip_address = models.CharField(max_length=100, blank=True, null=True, db_index=True)
+    port = models.IntegerField(default=0, db_index=True)
+    is_active = models.BooleanField(default=False, db_index=True)
+    last_checked = models.DateTimeField(auto_now=True)
+    last_active = models.DateTimeField(blank=True, default=now)
+    times_checked = models.IntegerField(default=0)
+    times_check_succeeded = models.IntegerField(default=0)
+
+    def __str__(self):
+        return f"{self.location} ({self.url})"
+
+
+class Subscription(models.Model):
+    class SubscriptionKind(models.TextChoices):
+        PLAIN = "PLAIN", "plain"
+        BASE64 = "BASE64", "base64"
+
+    url = models.URLField(null=False, unique=True)
+    kind = models.CharField(choices=SubscriptionKind.choices, default=SubscriptionKind.PLAIN, max_length=10)
+    alive = models.BooleanField(default=True)
+    alive_timestamp = models.DateTimeField(default=now)
+    enabled = models.BooleanField(default=True)
+    error_message = models.CharField(max_length=10000, default="")
+
+    def __str__(self):
+        return f"{self.url} - {self.kind}"
+
+
+class TaskLog(models.Model):
+    name = models.CharField(max_length=100)
+    details = models.CharField(max_length=1000, default="")
+    start_time = models.DateTimeField(auto_now=False)
+    finish_time = models.DateTimeField(auto_now=False)
+
+    def __str__(self):
+        return f"{self.name} - {self.finish_time}"

app/shadowmere/apps/proxylist/pagination.py

@@ -0,0 +1,14 @@
+from rest_framework import pagination
+from rest_framework.response import Response
+
+
+class ProxiesPagination(pagination.PageNumberPagination):
+    def get_paginated_response(self, data):
+        return Response(
+            {
+                "count": self.page.paginator.count,
+                "total_pages": self.page.paginator.num_pages,
+                "current_page": self.page.number,
+                "results": data,
+            }
+        )

proxylist/permissions.py → app/shadowmere/apps/proxylist/permissions.py

@@ -1,4 +1,4 @@
-from rest_framework.permissions import BasePermission, SAFE_METHODS
+from rest_framework.permissions import SAFE_METHODS, BasePermission
 
 
 class GeneralPermission(BasePermission):