added robots, sitemaps

akarpov/about/sitemaps.py

@@ -0,0 +1,14 @@
+from django.contrib.sitemaps import Sitemap
+
+from akarpov.about.models import Project
+
+
+class ProjectsSitemap(Sitemap):
+    changefreq = "never"
+    priority = 1
+
+    def items(self):
+        return Project.objects.all()
+
+    def lastmod(self, obj):
+        return obj.modified

akarpov/blog/migrations/0006_post_public.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2 on 2023-04-20 09:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("blog", "0005_alter_post_slug"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="post",
+            name="public",
+            field=models.BooleanField(default=True),
+        ),
+    ]

akarpov/blog/models.py

@@ -24,6 +24,7 @@ class Post(BaseImageModel, ShortLink):
 
     created = models.DateTimeField(auto_now_add=True)
     edited = models.DateTimeField(auto_now=True)
+    public = models.BooleanField(default=True)
 
     tags = models.ManyToManyField("blog.Tag", related_name="posts")
 

akarpov/blog/sitemaps.py

@@ -0,0 +1,14 @@
+from django.contrib.sitemaps import Sitemap
+
+from akarpov.blog.models import Post
+
+
+class BlogSitemap(Sitemap):
+    changefreq = "never"
+    priority = 1
+
+    def items(self):
+        return Post.objects.filter(public=True)
+
+    def lastmod(self, obj):
+        return obj.edited

akarpov/files/previews/__init__.py

@@ -8,6 +8,7 @@
         "doc": application.doc.view,
         "docx": application.docx.view,
         "vnd.oasis.opendocument.text": application.odt.view,
+        "x-httpd-php": text.common.view,
     },
     "audio": {
         "aac": audio.basic.view,

akarpov/files/sitemaps.py

@@ -0,0 +1,14 @@
+from django.contrib.sitemaps import Sitemap
+
+from akarpov.files.models import File
+
+
+class FileSitemap(Sitemap):
+    changefreq = "never"
+    priority = 0.8
+
+    def items(self):
+        return File.objects.filter(private=False)
+
+    def lastmod(self, obj):
+        return obj.modified

akarpov/users/sitemaps.py

@@ -0,0 +1,14 @@
+from django.contrib.sitemaps import Sitemap
+
+from akarpov.users.models import User
+
+
+class UserSitemap(Sitemap):
+    changefreq = "never"
+    priority = 0.5
+
+    def items(self):
+        return User.objects.all()
+
+    def lastmod(self, obj):
+        return obj.last_login

config/settings/base.py

@@ -84,6 +84,7 @@
     "django.contrib.contenttypes",
     "django.contrib.sessions",
     "django.contrib.sites",
+    "django.contrib.sitemaps",
     "django.contrib.messages",
     "django.contrib.staticfiles",
     "django.contrib.humanize",  # Handy template tags
@@ -113,6 +114,7 @@
     "extra_settings",
     "akarpov.contrib.chunked_upload",
     "active_link",
+    "robots",
     # django-cms
     "cms",
     "menus",
@@ -538,3 +540,8 @@
 
 # YANDEX_MUSIC
 MUSIC_YANDEX_TOKEN = env("MPD_HOST", default="")
+
+# ROBOTS
+# ------------------------------------------------------------------------------
+ROBOTS_USE_SITEMAP = True
+ROBOTS_USE_SCHEME_IN_HOST = True

config/sitemaps.py

@@ -0,0 +1,11 @@
+from akarpov.about.sitemaps import ProjectsSitemap
+from akarpov.blog.sitemaps import BlogSitemap
+from akarpov.files.sitemaps import FileSitemap
+from akarpov.users.sitemaps import UserSitemap
+
+sitemaps = {
+    "static": BlogSitemap,
+    "projects": ProjectsSitemap,
+    "files": FileSitemap,
+    "users": UserSitemap,
+}

config/urls.py

@@ -1,7 +1,8 @@
 from django.conf import settings
 from django.conf.urls.static import static
 from django.contrib import admin
-from django.urls import include, path
+from django.contrib.sitemaps.views import sitemap
+from django.urls import include, path, re_path
 from django.views import defaults as default_views
 from django.views.generic import TemplateView
 from drf_spectacular.views import (
@@ -12,9 +13,17 @@
 
 from akarpov.about.views import about_view
 from akarpov.tools.shortener.views import redirect_view
+from config.sitemaps import sitemaps
 
 urlpatterns = [
     path("home", TemplateView.as_view(template_name="pages/home.html"), name="home"),
+    re_path(r"^robots\.txt", include("robots.urls")),
+    path(
+        "sitemap.xml",
+        sitemap,
+        {"sitemaps": sitemaps},
+        name="django.contrib.sitemaps.views.sitemap",
+    ),
     path("health/", include("health_check.urls")),
     path("cms/", include("cms.urls")),
     # Django Admin, use {% url 'admin:index' %}

poetry.lock

@@ -1666,6 +1666,17 @@ six = ">=1.9.0"
 [package.extras]
 with-social = ["django-allauth (>=0.25.0)"]
 
+[[package]]
+name = "django-robots"
+version = "5.0"
+description = "Robots exclusion application for Django, complementing Sitemaps."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "django-robots-5.0.tar.gz", hash = "sha256:f5817975a9e3b082440a81396111d8a6671b54f6aa11349847554bd1beebd959"},
+]
+
 [[package]]
 name = "django-sekizai"
 version = "4.0.0"
@@ -3408,7 +3419,6 @@ files = [
     {file = "Pillow-9.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c1170d6b195555644f0616fd6ed929dfcf6333b8675fcca044ae5ab110ded296"},
     {file = "Pillow-9.5.0-cp311-cp311-win32.whl", hash = "sha256:54f7102ad31a3de5666827526e248c3530b3a33539dbda27c6843d19d72644ec"},
     {file = "Pillow-9.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:cfa4561277f677ecf651e2b22dc43e8f5368b74a25a8f7d1d4a3a243e573f2d4"},
-    {file = "Pillow-9.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:965e4a05ef364e7b973dd17fc765f42233415974d773e82144c9bbaaaea5d089"},
     {file = "Pillow-9.5.0-cp312-cp312-win32.whl", hash = "sha256:22baf0c3cf0c7f26e82d6e1adf118027afb325e703922c8dfc1d5d0156bb2eeb"},
     {file = "Pillow-9.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:432b975c009cf649420615388561c0ce7cc31ce9b2e374db659ee4f7d57a1f8b"},
     {file = "Pillow-9.5.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:5d4ebf8e1db4441a55c509c4baa7a0587a0210f7cd25fcfe74dbbce7a4bd1906"},
@@ -5289,6 +5299,7 @@ category = "main"
 optional = false
 python-versions = "*"
 files = [
+    {file = "wcwidth-0.2.6-py2.py3-none-any.whl", hash = "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e"},
     {file = "wcwidth-0.2.6.tar.gz", hash = "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"},
 ]
 
@@ -5631,4 +5642,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.11"
-content-hash = "20cd7aa75803b18a50189bf57c0f26a103d737d98f9783385f3cf6f1aa82044f"
+content-hash = "667d1226a682d79bea11fdee05d079b1b494ac325167ca069f79f4d18d1a7d04"

pyproject.toml

@@ -87,6 +87,7 @@ ffmpeg-python = "^0.2.0"
 cairosvg = "^2.7.0"
 textract = "^1.6.5"
 spotipy = "2.16.1"
+django-robots = "^5.0"
 
 
 [build-system]