Merge pull request #612 from Mailu/feat-abstract-db

Abstract db access from Postfix and Dovecot

core/admin/mailu/__init__.py

@@ -11,7 +11,6 @@ import os
 import docker
 import socket
 import uuid
-import redis
 
 from werkzeug.contrib import fixers
 
@@ -89,9 +88,6 @@ manager.add_command('db', flask_migrate.MigrateCommand)
 babel = flask_babel.Babel(app)
 translations = list(map(str, babel.list_translations()))
 
-# Quota manager
-quota = redis.Redis.from_url(app.config.get("QUOTA_STORAGE_URL"))
-
 @babel.localeselector
 def get_locale():
     return flask.request.accept_languages.best_match(translations)

core/admin/mailu/internal/__init__.py

@@ -6,7 +6,8 @@ import socket
 import flask
 
 
-internal = flask.Blueprint('internal', __name__)
+internal = flask.Blueprint('internal', __name__, template_folder='templates')
+
 
 @internal.app_errorhandler(RateLimitExceeded)
 def rate_limit_handler(e):
@@ -17,6 +18,7 @@ def rate_limit_handler(e):
         response.headers['Auth-Wait'] = '3'
     return response
 
+
 @limiter.request_filter
 def whitelist_webmail():
     try:
@@ -26,4 +28,4 @@ def whitelist_webmail():
         return False
 
 
-from mailu.internal import views
+from mailu.internal.views import *

core/admin/mailu/internal/templates/default.sieve

@@ -8,8 +8,6 @@ require "regex";
 require "relational";
 require "date";
 require "comparator-i;ascii-numeric";
-require "vnd.dovecot.extdata";
-require "vnd.dovecot.execute";
 require "spamtestplus";
 require "editheader";
 require "index";
@@ -20,21 +18,23 @@ if header :index 2 :matches "Received" "from * by * for <*>; *"
   addheader "Delivered-To" "<${3}>";
 }
 
-if allof (string :is "${extdata.spam_enabled}" "1",
+{% if user.spam_enabled %}
-          spamtest :percent :value "gt" :comparator "i;ascii-numeric"  "${extdata.spam_threshold}")
+if spamtest :percent :value "gt" :comparator "i;ascii-numeric"  "{{ user.spam_threshold }}"
 {
   setflag "\\seen";
   fileinto :create "Junk";
   stop;
 }
+{% endif %}
 
 if exists "X-Virus" {
   discard;
   stop;
 }
 
-if allof (string :is "${extdata.reply_enabled}" "1",
+{% if user.reply_enabled %}
-          currentdate :value "le" "date" "${extdata.reply_enddate}")
+if currentdate :value "le" "date" "{{ user.reply_enddate }}"
 {
-  vacation :days 1 :subject "${extdata.reply_subject}" "${extdata.reply_body}";
+ vacation :days 1 :subject "{{ user.reply_subject }}" "{{ user.reply_body }}";
 }
+{% endif %}

core/admin/mailu/internal/views/__init__.py

@@ -0,0 +1,3 @@
+__all__ = [
+    'auth', 'postfix', 'dovecot', 'fetch'
+]

core/admin/mailu/internal/views/auth.py

@@ -4,7 +4,6 @@ from mailu.internal import internal, nginx
 import flask
 import flask_login
 import base64
-import urllib
 
 
 @internal.route("/auth/email")

core/admin/mailu/internal/views/dovecot.py

@@ -0,0 +1,40 @@
+from mailu import db, models
+from mailu.internal import internal
+
+import flask
+
+
+@internal.route("/dovecot/passdb/<user_email>")
+def dovecot_passdb_dict(user_email):
+    user = models.User.query.get(user_email) or flask.abort(404)
+    return flask.jsonify({
+        "password": user.password,
+    })
+
+
+@internal.route("/dovecot/userdb/<user_email>")
+def dovecot_userdb_dict(user_email):
+    user = models.User.query.get(user_email) or flask.abort(404)
+    return flask.jsonify({
+        "quota_rule": "*:bytes={}".format(user.quota_bytes)
+    })
+
+
+@internal.route("/dovecot/quota/<ns>/<user_email>", methods=["POST"])
+def dovecot_quota(ns, user_email):
+    user = models.User.query.get(user_email) or flask.abort(404)
+    if ns == "storage":
+        user.quota_bytes_used = flask.request.get_json()
+        db.session.commit()
+    return flask.jsonify(None)
+
+
+@internal.route("/dovecot/sieve/name/<script>/<user_email>")
+def dovecot_sieve_name(script, user_email):
+    return flask.jsonify(script)
+
+
+@internal.route("/dovecot/sieve/data/default/<user_email>")
+def dovecot_sieve_data(user_email):
+    user = models.User.query.get(user_email) or flask.abort(404)
+    return flask.jsonify(flask.render_template("default.sieve", user=user))

core/admin/mailu/internal/views/fetch.py

@@ -0,0 +1,32 @@
+from mailu import db, models
+from mailu.internal import internal
+
+import flask
+import datetime
+
+
+@internal.route("/fetch")
+def fetch_list():
+    return flask.jsonify([
+        {
+            "id": fetch.id,
+            "tls": fetch.tls,
+            "keep": fetch.keep,
+            "user_email": fetch.user_email,
+            "protocol": fetch.protocol,
+            "host": fetch.host,
+            "port": fetch.port,
+            "username": fetch.username,
+            "password": fetch.password
+        } for fetch in models.Fetch.query.all()
+    ])
+
+
+@internal.route("/fetch/<fetch_id>", methods=["POST"])
+def fetch_done(fetch_id):
+    fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
+    fetch.last_check = datetime.datetime.now()
+    fetch.error_message = str(flask.request.get_json())
+    db.session.add(fetch)
+    db.session.commit()
+    return ""

core/admin/mailu/internal/views/postfix.py

@@ -0,0 +1,54 @@
+from mailu import db, models
+from mailu.internal import internal
+
+import flask
+
+
+@internal.route("/postfix/domain/<domain_name>")
+def postfix_mailbox_domain(domain_name):
+    domain = models.Domain.query.get(domain_name) or flask.abort(404)
+    return flask.jsonify(domain.name)
+
+
+@internal.route("/postfix/mailbox/<email>")
+def postfix_mailbox_map(email):
+    user = models.User.query.get(email) or flask.abort(404)
+    return flask.jsonify(user.email)
+
+
+@internal.route("/postfix/alias/<alias>")
+def postfix_alias_map(alias):
+    localpart, domain = alias.split('@', 1) if '@' in alias else (None, alias)
+    alternative = models.Alternative.query.get(domain)
+    if alternative:
+        domain = alternative.domain_name
+    email = '{}@{}'.format(localpart, domain)
+    if localpart is None:
+        return flask.jsonify(domain)
+    else:
+        alias_obj = models.Alias.resolve(localpart, domain)
+        if alias_obj:
+            return flask.jsonify(",".join(alias_obj.destination))
+        user_obj = models.User.query.get(email)
+        if user_obj:
+            return flask.jsonify(user_obj.destination)
+        return flask.abort(404)
+
+
+@internal.route("/postfix/transport/<email>")
+def postfix_transport(email):
+    localpart, domain = email.split('@', 1) if '@' in email else (None, email)
+    relay = models.Relay.query.get(domain) or flask.abort(404)
+    return flask.jsonify("smtp:[{}]".format(relay.smtp))
+
+
+@internal.route("/postfix/sender/<sender>")
+def postfix_sender(sender):
+    """ Simply reject any sender that pretends to be from a local domain
+    """
+    localpart, domain_name = sender.split('@', 1) if '@' in sender else (None, sender)
+    domain = models.Domain.query.get(domain_name)
+    alternative = models.Alternative.query.get(domain_name)
+    if domain or alternative:
+        return flask.jsonify("REJECT")
+    return flask.abort(404)

core/admin/mailu/models.py

@@ -1,11 +1,11 @@
-from mailu import app, db, dkim, login_manager, quota
+from mailu import app, db, dkim, login_manager
 
 from sqlalchemy.ext import declarative
 from passlib import context, hash
 from datetime import datetime, date
 from email.mime import text
 
-
+import sqlalchemy
 import re
 import time
 import os
@@ -235,6 +235,7 @@ class User(Base, Email):
         backref=db.backref('users', cascade='all, delete-orphan'))
     password = db.Column(db.String(255), nullable=False)
     quota_bytes = db.Column(db.Integer(), nullable=False, default=10**9)
+    quota_bytes_used = db.Column(db.Integer(), nullable=False, default=0)
     global_admin = db.Column(db.Boolean(), nullable=False, default=False)
     enabled = db.Column(db.Boolean(), nullable=False, default=True)
 
@@ -266,8 +267,14 @@ class User(Base, Email):
         return self.email
 
     @property
-    def quota_bytes_used(self):
+    def destination(self):
-        return quota.get(self.email + "/quota/storage") or 0
+        if self.forward_enabled:
+            result = self.self.forward_destination
+            if self.forward_keep:
+                result += ',' + self.email
+            return result
+        else:
+            return self.email
 
     scheme_dict = {'SHA512-CRYPT': "sha512_crypt",
                    'SHA256-CRYPT': "sha256_crypt",
@@ -329,6 +336,22 @@ class Alias(Base, Email):
     wildcard = db.Column(db.Boolean(), nullable=False, default=False)
     destination = db.Column(CommaSeparatedList, nullable=False, default=[])
 
+    @classmethod
+    def resolve(cls, localpart, domain_name):
+        return cls.query.filter(
+            sqlalchemy.and_(cls.domain_name == domain_name,
+                sqlalchemy.or_(
+                    sqlalchemy.and_(
+                        cls.wildcard == False,
+                        cls.localpart == localpart
+                    ), sqlalchemy.and_(
+                        cls.wildcard == True,
+                        sqlalchemy.bindparam("l", localpart).like(cls.localpart)
+                    )
+                )
+            )
+        ).first()
+
 
 class Token(Base):
     """ A token is an application password for a given user.

core/admin/migrations/versions/25fd6c7bcb4a_.py

@@ -0,0 +1,28 @@
+""" Add a column for used quota
+
+Revision ID: 25fd6c7bcb4a
+Revises: 049fed905da7
+Create Date: 2018-07-25 21:56:09.729153
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '25fd6c7bcb4a'
+down_revision = '049fed905da7'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    with op.batch_alter_table('user') as batch:
+        batch.add_column(sa.Column('quota_bytes_used', sa.Integer(), nullable=False, server_default='0'))
+
+
+def downgrade():
+    with op.batch_alter_table('user') as batch:
+        batch.drop_column('user', 'quota_bytes_used')

core/dovecot/Dockerfile

@@ -1,14 +1,12 @@
-FROM alpine:3.7
+FROM alpine:3.8
 
-RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
+RUN apk add --no-cache \
- && apk add --no-cache \
+     dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \
-     dovecot dovecot-sqlite dovecot-pigeonhole-plugin dovecot-pigeonhole-plugin-extdata \
+     python3 py3-pip \
-     dovecot-fts-lucene rspamd-client@testing python py-jinja2 py-pip \
+ && pip3 install --upgrade pip \
- && pip install --upgrade pip \
+ && pip3 install jinja2 podop tenacity
- && pip install tenacity
 
 COPY conf /conf
-COPY sieve /var/lib/dovecot
 COPY start.py /start.py
 
 EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp

core/dovecot/conf/auth.conf

@@ -0,0 +1,5 @@
+uri = proxy:/tmp/podop.socket:auth
+iterate_disable = yes
+default_pass_scheme = plain
+password_key = passdb/%u
+user_key = userdb/%u

core/dovecot/conf/dovecot-sql.conf.ext

@@ -1,18 +0,0 @@
-driver = sqlite
-connect = /data/main.db
-
-# Return the user hashed password
-password_query = \
- SELECT NULL as password, 'Y' as nopassword, '{% if POD_ADDRESS_RANGE %}{{ POD_ADDRESS_RANGE }}{% else %}{{ FRONT_ADDRESS }}{% if WEBMAIL_ADDRESS %},{{ WEBMAIL_ADDRESS }}{% endif %}{% endif %}' as allow_nets \
-   FROM user \
-  WHERE user.email = '%u'
-
-# Mostly get the user quota
-user_query = \
- SELECT '*:bytes=' || user.quota_bytes AS quota_rule \
-   FROM user \
-  WHERE user.email = '%u'
-
-# For using doveadm -A:
-iterate_query = \
- SELECT user.email AS user FROM user

core/dovecot/conf/dovecot.conf

@@ -7,17 +7,6 @@ postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }}
 hostname = {{ HOSTNAMES.split(",")[0] }}
 submission_host = {{ FRONT_ADDRESS }}
 
-service dict {
-  unix_listener dict {
-    group = mail
-    mode = 0660
-  }
-}
-
-dict {
-  sieve = sqlite:/etc/dovecot/pigeonhole-sieve.dict
-}
-
 ###############
 # Full-text search
 ###############
@@ -50,28 +39,18 @@ mail_plugins = $mail_plugins quota quota_clone zlib
 
 namespace inbox {
   inbox = yes
-  mailbox Trash {
+  {% for mailbox in ("Trash", "Drafts", "Sent", "Junk") %}
+  mailbox {{ mailbox }} {
     auto = subscribe
-    special_use = \Trash
+    special_use = \{{ mailbox }}
-  }
-  mailbox Drafts {
-    auto = subscribe
-    special_use = \Drafts
-  }
-  mailbox Sent {
-    auto = subscribe
-    special_use = \Sent
-  }
-  mailbox Junk {
-    auto = subscribe
-    special_use = \Junk
   }
+  {% endfor %}
 }
 
 plugin {
   quota = count:User quota
   quota_vsizes = yes
-  quota_clone_dict = redis:host={{ REDIS_ADDRESS }}:port=6379:db=1
+  quota_clone_dict = proxy:/tmp/podop.socket:quota
 
   {% if COMPRESSION in [ 'gz', 'bz2' ] %}
   zlib_save = {{ COMPRESSION }}
@@ -87,16 +66,15 @@ plugin {
 ###############
 auth_mechanisms = plain login
 disable_plaintext_auth = no
-ssl_protocols = !SSLv3
 
 passdb {
-  driver = sql
+  driver = dict
-  args = /etc/dovecot/dovecot-sql.conf.ext
+  args = /etc/dovecot/auth.conf
 }
 
 userdb {
-  driver = sql
+  driver = dict
-  args = /etc/dovecot/dovecot-sql.conf.ext
+  args = /etc/dovecot/auth.conf
 }
 
 service auth {
@@ -117,7 +95,6 @@ service auth-worker {
 ###############
 # IMAP & POP
 ###############
-
 protocol imap {
   mail_plugins = $mail_plugins imap_quota imap_sieve
 }
@@ -135,7 +112,6 @@ service imap-login {
 ###############
 # Delivery
 ###############
-
 protocol lmtp {
   mail_plugins = $mail_plugins sieve
   recipient_delimiter = {{ RECIPIENT_DELIMITER }}
@@ -147,11 +123,9 @@ service lmtp {
   }
 }
 
-
 ###############
 # Filtering
 ###############
-
 service managesieve-login {
   inet_listener sieve {
     port = 4190
@@ -162,16 +136,13 @@ service managesieve {
 }
 
 plugin {
-  sieve = file:~/sieve;active=~/.dovecot.sieve
+  sieve = dict:proxy:/tmp/podop.socket:sieve
-  sieve_plugins = sieve_extdata sieve_imapsieve sieve_extprograms
+  sieve_plugins = sieve_imapsieve sieve_extprograms
-  sieve_global_extensions = +vnd.dovecot.extdata +spamtest +spamtestplus +vnd.dovecot.execute +editheader
+  sieve_extensions = +spamtest +spamtestplus +editheader
-  sieve_before = /var/lib/dovecot/before.sieve
+  sieve_global_extensions = +vnd.dovecot.execute
-  sieve_default = /var/lib/dovecot/default.sieve
-  sieve_after = /var/lib/dovecot/after.sieve
-  sieve_extdata_dict_uri = proxy::sieve
 
   # Sieve execute
-  sieve_execute_bin_dir = /var/lib/dovecot/bin
+  sieve_execute_bin_dir = /conf/bin
 
   # Send vacation replies even for aliases
   # See the Pigeonhole documentation about warnings: http://wiki2.dovecot.org/Pigeonhole/Sieve/Extensions/Vacation
@@ -190,11 +161,11 @@ plugin {
   # Learn from spam
   imapsieve_mailbox1_name = Junk
   imapsieve_mailbox1_causes = COPY
-  imapsieve_mailbox1_before = file:/var/lib/dovecot/report-spam.sieve
+  imapsieve_mailbox1_before = file:/conf/report-spam.sieve
   imapsieve_mailbox2_name = *
   imapsieve_mailbox2_from = Junk
   imapsieve_mailbox2_causes = COPY
-  imapsieve_mailbox2_before = file:/var/lib/dovecot/report-ham.sieve
+  imapsieve_mailbox2_before = file:/conf/report-ham.sieve
 }
 
 ###############

core/dovecot/conf/pigeonhole-sieve.dict

@@ -1,43 +0,0 @@
-connect = /data/main.db
-
-map {
-  pattern = priv/spam_enabled
-  table = user
-  username_field = email
-  value_field = spam_enabled
-}
-
-map {
-  pattern = priv/spam_threshold
-  table = user
-  username_field = email
-  value_field = spam_threshold
-}
-
-map {
-  pattern = priv/reply_enabled
-  table = user
-  username_field = email
-  value_field = reply_enabled
-}
-
-map {
-  pattern = priv/reply_subject
-  table = user
-  username_field = email
-  value_field = reply_subject
-}
-
-map {
-  pattern = priv/reply_body
-  table = user
-  username_field = email
-  value_field = reply_body
-}
-
-map {
-  pattern = priv/reply_enddate
-  table = user
-  username_field = email
-  value_field = reply_enddate
-}

core/dovecot/start.py

@@ -1,11 +1,23 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 import jinja2
 import os
 import socket
 import glob
+import multiprocessing
 import tenacity
+
 from tenacity import retry
+from podop import run_server
+
+
+def start_podop():
+    os.setuid(8)
+    run_server(3 if "DEBUG" in os.environ else 0, "dovecot", "/tmp/podop.socket", [
+		("quota", "url", "http://admin/internal/dovecot/§"),
+		("auth", "url", "http://admin/internal/dovecot/§"),
+		("sieve", "url", "http://admin/internal/dovecot/§"),
+    ])
 
 convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
 
@@ -18,9 +30,11 @@ def resolve():
 
 # Actual startup script
 resolve()
-for dovecot_file in glob.glob("/conf/*"):
+
+for dovecot_file in glob.glob("/conf/*.conf"):
     convert(dovecot_file, os.path.join("/etc/dovecot", os.path.basename(dovecot_file)))
 
-# Run postfix
+# Run Podop, then postfix
+multiprocessing.Process(target=start_podop).start()
 os.system("chown -R mail:mail /mail /var/lib/dovecot")
 os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"])

core/postfix/Dockerfile

@@ -1,8 +1,9 @@
-FROM alpine:3.7
+FROM alpine:3.8
 
-RUN apk add --no-cache postfix postfix-sqlite postfix-pcre rsyslog python py-jinja2 py-pip \
+RUN apk add --no-cache postfix postfix-pcre rsyslog \
- && pip install --upgrade pip \
+    python3 py3-pip \
- && pip install tenacity
+ && pip3 install --upgrade pip \
+ && pip3 install jinja2 podop tenacity
 
 COPY conf /conf
 COPY start.py /start.py

core/postfix/conf/main.cf

@@ -2,6 +2,8 @@
 # General
 ###############
 
+debug_peer_list = 0.0.0.0/0
+
 # Main domain and hostname
 mydomain = {{ DOMAIN }}
 myhostname = {{ HOSTNAMES.split(",")[0] }}
@@ -19,8 +21,8 @@ mynetworks = 127.0.0.1/32 [::1]/128 {{ RELAYNETS }}
 # Empty alias list to override the configuration variable and disable NIS
 alias_maps =
 
-# SQLite configuration
+# Podop configuration
-sql = sqlite:${config_directory}/
+podop = socketmap:unix:/tmp/podop.socket:
 
 # Only accept virtual emails
 mydestination =
@@ -56,13 +58,14 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
 
 # The alias map actually returns both aliases and local mailboxes, which is
 # required for reject_unlisted_sender to work properly
-virtual_alias_maps = ${sql}sqlite-virtual_alias_maps.cf
+virtual_alias_domains =
-virtual_mailbox_domains = ${sql}sqlite-virtual_mailbox_domains.cf
+virtual_alias_maps = ${podop}alias
-virtual_mailbox_maps = $virtual_alias_maps
+virtual_mailbox_domains = ${podop}domain
+virtual_mailbox_maps = ${podop}mailbox
 
 # Mails are transported if required, then forwarded to Dovecot for delivery
-relay_domains = ${sql}sqlite-transport.cf
+relay_domains = ${podop}transport
-transport_maps = ${sql}sqlite-transport.cf
+transport_maps = ${podop}transport
 virtual_transport = lmtp:inet:{{ HOST_LMTP }}
 
 # In order to prevent Postfix from running DNS query, enforce the use of the
@@ -82,15 +85,20 @@ smtpd_sender_login_maps = $virtual_alias_maps
 # Restrictions for incoming SMTP, other restrictions are applied in master.cf
 smtpd_helo_required = yes
 
-smtpd_recipient_restrictions =
+smtpd_client_restrictions =
   permit_mynetworks,
-  check_sender_access ${sql}sqlite-reject-spoofed.cf,
+  check_sender_access ${podop}sender,
   reject_non_fqdn_sender,
   reject_unknown_sender_domain,
   reject_unknown_recipient_domain,
   reject_unverified_recipient,
   permit
 
+smtpd_relay_restrictions =
+  permit_mynetworks,
+  permit_sasl_authenticated,
+  reject_unauth_destination
+
 unverified_recipient_reject_reason = Address lookup failure
 
 ###############

core/postfix/conf/master.cf

@@ -7,7 +7,7 @@ smtp      inet  n       -       n       -       -       smtpd
 # Internal SMTP service
 10025     inet  n       -       n       -       -       smtpd
   -o smtpd_sasl_auth_enable=yes
-  -o smtpd_recipient_restrictions=reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit
+  -o smtpd_client_restrictions=reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit
   -o smtpd_reject_unlisted_recipient={% if REJECT_UNLISTED_RECIPIENT %}{{ REJECT_UNLISTED_RECIPIENT }}{% else %}no{% endif %}
   -o cleanup_service_name=outclean
 outclean  unix n       -       n       -       0       cleanup

core/postfix/conf/sqlite-reject-spoofed.cf

@@ -1,5 +0,0 @@
-dbpath = /data/main.db
-query =
-  SELECT 'REJECT' FROM domain WHERE name='%s'
-  UNION
-  SELECT 'REJECT' FROM alternative WHERE name='%s'

core/postfix/conf/sqlite-transport.cf

@@ -1,3 +0,0 @@
-dbpath = /data/main.db
-query =
-  SELECT 'smtp:['||smtp||']' FROM relay WHERE name='%s'

core/postfix/conf/sqlite-virtual_alias_maps.cf

@@ -1,23 +0,0 @@
-dbpath = /data/main.db
-query =
-  SELECT destination
-   FROM
-     (SELECT destination, email, wildcard, localpart, localpart||'@'||alternative.name AS alt_email FROM alias LEFT JOIN alternative ON alias.domain_name = alternative.domain_name
-      UNION
-      SELECT (CASE WHEN forward_enabled=1 THEN (CASE WHEN forward_keep=1 THEN email||',' ELSE '' END)||forward_destination ELSE email END) AS destination, email, 0 as wildcard, localpart, localpart||'@'||alternative.name as alt_email FROM user LEFT JOIN alternative ON user.domain_name = alternative.domain_name
-      UNION
-      SELECT '@'||domain_name as destination, '@'||name as email, 0 as wildcard, '' as localpart, NULL AS alt_email FROM alternative)
-   WHERE
-     (
-      wildcard = 0
-      AND
-      (email = '%s' OR alt_email = '%s')
-     ) OR (
-      wildcard = 1
-      AND
-      '%s' LIKE email
-     )
-   ORDER BY
-     wildcard ASC,
-     length(localpart) DESC
-   LIMIT 1

core/postfix/conf/sqlite-virtual_mailbox_domains.cf

@@ -1,5 +0,0 @@
-dbpath = /data/main.db
-query =
-  SELECT name FROM domain WHERE name='%s'
-  UNION
-  SELECT name FROM alternative WHERE name='%s'

core/postfix/start.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 import jinja2
 import os
@@ -6,7 +6,21 @@ import socket
 import glob
 import shutil
 import tenacity
+import multiprocessing
+
 from tenacity import retry
+from podop import run_server
+
+
+def start_podop():
+    os.setuid(100)
+    run_server(3 if "DEBUG" in os.environ else 0, "postfix", "/tmp/podop.socket", [
+		("transport", "url", "http://admin/internal/postfix/transport/§"),
+		("alias", "url", "http://admin/internal/postfix/alias/§"),
+		("domain", "url", "http://admin/internal/postfix/domain/§"),
+        ("mailbox", "url", "http://admin/internal/postfix/mailbox/§"),
+        ("sender", "url", "http://admin/internal/postfix/sender/§")
+    ])
 
 convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
 
@@ -38,7 +52,8 @@ for map_file in glob.glob("/overrides/*.map"):
 
 convert("/conf/rsyslog.conf", "/etc/rsyslog.conf")
 
-# Run postfix
+# Run Podop and Postfix
+multiprocessing.Process(target=start_podop).start()
 if os.path.exists("/var/run/rsyslogd.pid"):
     os.remove("/var/run/rsyslogd.pid")
 os.system("/usr/lib/postfix/post-install meta_directory=/etc/postfix create-missing")

docs/compose/docker-compose.yml

@@ -39,7 +39,6 @@ services:
     restart: always
     env_file: .env
     volumes:
-      - "$ROOT/data:/data"
       - "$ROOT/mail:/mail"
       - "$ROOT/overrides:/overrides"
     depends_on:
@@ -50,7 +49,6 @@ services:
     restart: always
     env_file: .env
     volumes:
-      - "$ROOT/data:/data"
       - "$ROOT/overrides:/overrides"
     depends_on:
       - front
@@ -104,5 +102,3 @@ services:
     image: mailu/fetchmail:$VERSION
     restart: always
     env_file: .env
-    volumes:
-      - "$ROOT/data:/data"

services/fetchmail/Dockerfile

@@ -1,7 +1,9 @@
-FROM python:alpine
+FROM python:3-alpine
 
-RUN apk add --no-cache fetchmail ca-certificates
+RUN apk add --no-cache fetchmail ca-certificates \
+ && pip install requests
 
 COPY fetchmail.py /fetchmail.py
+USER fetchmail
 
 CMD ["/fetchmail.py"]

services/fetchmail/fetchmail.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python
 
-import sqlite3
 import time
 import os
 import tempfile
 import shlex
 import subprocess
 import re
+import requests
 
 
 FETCHMAIL = """
@@ -15,6 +15,7 @@ fetchmail -N \
     -f {}
 """
 
+
 RC_LINE = """
 poll "{host}" proto {protocol}  port {port}
     user "{username}" password "{password}"
@@ -24,10 +25,12 @@ poll "{host}" proto {protocol}  port {port}
     sslproto 'AUTO'
 """
 
+
 def extract_host_port(host_and_port, default_port):
     host, _, port = re.match('^(.*)(:([0-9]*))?$', host_and_port).groups()
     return host, int(port) if port else default_port
 
+
 def escape_rc_string(arg):
     return arg.replace("\\", "\\\\").replace('"', '\\"')
 
@@ -41,30 +44,26 @@ def fetchmail(fetchmailrc):
         return output
 
 
-def run(connection, cursor, debug):
+def run(debug):
-    cursor.execute("""
+    fetches = requests.get("http://admin/internal/fetch").json()
-        SELECT user_email, protocol, host, port, tls, username, password, keep
-        FROM fetch
-    """)
     smtphost, smtpport = extract_host_port(os.environ.get("HOST_SMTP", "smtp"), None)
     if smtpport is None:
         smtphostport = smtphost
     else:
         smtphostport = "%s/%d" % (smtphost, smtpport)
-    for line in cursor.fetchall():
+    for fetch in fetches:
         fetchmailrc = ""
-        user_email, protocol, host, port, tls, username, password, keep = line
         options = "options antispam 501, 504, 550, 553, 554"
-        options += " ssl" if tls else ""
+        options += " ssl" if fetch["tls"] else ""
-        options += " keep" if keep else " fetchall"
+        options += " keep" if fetch["keep"] else " fetchall"
         fetchmailrc += RC_LINE.format(
-            user_email=escape_rc_string(user_email),
+            user_email=escape_rc_string(fetch["user_email"]),
-            protocol=protocol,
+            protocol=fetch["protocol"],
-            host=escape_rc_string(host),
+            host=escape_rc_string(fetch["host"]),
-            port=port,
+            port=fetch["port"],
             smtphost=smtphostport,
-            username=escape_rc_string(username),
+            username=escape_rc_string(fetch["username"]),
-            password=escape_rc_string(password),
+            password=escape_rc_string(fetch["password"]),
             options=options
         )
         if debug:
@@ -77,26 +76,20 @@ def run(connection, cursor, debug):
             # No mail is not an error
             if not error_message.startswith("fetchmail: No mail"):
                 print(error_message)
-            user_info = "for %s at %s" % (user_email, host)
+            user_info = "for %s at %s" % (fetch["user_email"], fetch["host"])
             # Number of messages seen is not a error as well
             if ("messages" in error_message and
                     "(seen " in error_message and
                     user_info in error_message):
                 print(error_message)
         finally:
-            cursor.execute("""
+            requests.post("http://admin/internal/fetch/{}".format(fetch["id"]),
-                UPDATE fetch SET error=?, last_check=datetime('now')
+                json=error_message.split("\n")[0]
-                WHERE user_email=?
+            )
-            """, (error_message.split("\n")[0], user_email))
-            connection.commit()
 
 
 if __name__ == "__main__":
-    debug = os.environ.get("DEBUG", None) == "True"
-    db_path = os.environ.get("DB_PATH", "/data/main.db")
-    connection = sqlite3.connect(db_path)
     while True:
-        cursor = connection.cursor()
-        run(connection, cursor, debug)
-        cursor.close()
         time.sleep(int(os.environ.get("FETCHMAIL_DELAY", 60)))
+        run(os.environ.get("DEBUG", None) == "True")
+

tests/smtp.py

@@ -0,0 +1,17 @@
+import smtplib
+import sys
+from email import mime
+
+from email.mime.image import MIMEImage
+from email.mime.multipart import MIMEMultipart
+
+msg = mime.multipart.MIMEMultipart()
+msg['Subject'] = 'Test email'
+msg['From'] = sys.argv[1]
+msg['To'] = sys.argv[2]
+msg.preamble = 'Test email'
+
+s = smtplib.SMTP('localhost')
+s.set_debuglevel(1)
+s.send_message(msg)
+s.quit()