v0.14.0 release

.github/workflows/python-package.yml

@@ -33,10 +33,6 @@ jobs:
         python -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-    - name: Echo env
-      run: |
-        echo ${{ github.event.pull_request.base.ref }}
-        echo ${{ github.event.pull_request.base.ref_name }}
     - name: Black and flake
       run: |
         bash run_lint.sh
@@ -48,19 +44,20 @@ jobs:
       run: |
         bash run_tests.sh full --test-group-count 3 --test-group=1 --reruns 3 --reruns-delay 15 
       if: github.event.pull_request.base.ref == 'main'
+      continue-on-error: true
     - name: Test changes with full tests 2/3
       run: |
         bash run_tests.sh full --test-group-count 3 --test-group=2 --reruns 3 --reruns-delay 15 
       if: github.event.pull_request.base.ref == 'main'
+      continue-on-error: true
     - name: Test changes with full tests 3/3
       run: |
         bash run_tests.sh full --test-group-count 3 --test-group=3 --reruns 3 --reruns-delay 15 
       if: github.event.pull_request.base.ref == 'main'
+      continue-on-error: true
     - name: Upload test results
       uses: actions/upload-artifact@v4
       with:
         name: Test result
         path: report.html
-
-
-
+      if: ${{ always() }}

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 Clement Julia
+Copyright (c) 2025 Clement Julia
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

docs/source/changelog.rst

@@ -8,6 +8,18 @@ The page attempt to keep a clear list of breaking/non-breaking changes and new f
    :local:
    :backlinks: none
 
+v0.14.0
+-----------
+Bug Fixes
+###########
+* Changed SLA adapter to CurlCffiAdapter
+* `Profile.follow` is more reliably parsed
+
+New Features
+#############
+* Exposed front page poll url under `FrontPage.poll_url`
+
+
 v0.13.0
 -----------
 Bug Fixes

docs/source/client.rst

@@ -13,6 +13,13 @@ Client
     :inherited-members:
 
 
+TwoFactorAuthClient
+--------------------
+.. autoclass:: moddb.client.TwoFactorAuthClient
+    :members:
+    :inherited-members:
+
+
 ThreadThumbnail
 ----------------
 .. autoclass:: moddb.client.ThreadThumbnail

docs/source/conf.py

@@ -21,13 +21,13 @@
 # -- Project information -----------------------------------------------------
 
 project = "moddb"
-copyright = "2022, Clement Julia"
+copyright = "2025, Clement Julia"
 author = "Clement Julia"
 
 # The short X.Y version
 version = ""
 # The full version, including alpha/beta/rc tags
-release = "0.13.0"
+release = "0.14.0"
 
 
 # -- General configuration ---------------------------------------------------

docs/source/index.rst

@@ -80,6 +80,22 @@ that mod page you would be forced to completly parse the result page on your own
 See more snippets there: :ref:`snippets-ref`.
 
 
+Two Factor Authentication
+--------------------------
+Sometimes ModDB will ask you for a code sent to your email when logging from a new device. There is no
+way to circuvement this. As such the library provides a class to do this handshake::
+
+   >> import moddb
+   >> e = moddb.TwoFactorAuthClient("MyUser", "*****")
+   >> e.login()
+   False
+   >> e.submit_2fa_code("AZADV")
+   < Member >
+
+
+This class' `login` method returns false if 2FA is required instead of erroring, allowing you elegantly 
+check for the code and to send it in a second request.
+
 Searching
 ----------
 

moddb/__init__.py

@@ -1,15 +1,17 @@
+from curl_adapter import CurlCffiAdapter
 import requests
 
 from .base import front_page, login, logout, parse_page, parse_results, rss, search, search_tags
-from .client import Client, Thread
+from .client import Client, TwoFactorAuthClient, Thread
 from .enums import *
 from .pages import *
-from .utils import BASE_URL, LOGGER, Object, get_page, request, soup, SSLAdapter
+from .utils import BASE_URL, LOGGER, Object, get_page, request, soup
 
 SESSION = requests.Session()
-SESSION.mount("https://", SSLAdapter())
+SESSION.mount("http://", CurlCffiAdapter())
+SESSION.mount("https://", CurlCffiAdapter())
 
-__version__ = "0.13.0"
+__version__ = "0.14.0"
 
 __all__ = [
     "front_page",
@@ -21,6 +23,7 @@
     "search",
     "search_tags",
     "Client",
+    "TwoFactorAuthClient",
     "Thread",
     "BASE_URL",
     "LOGGER",

moddb/boxes.py

@@ -208,19 +208,8 @@ def __init__(self, html: BeautifulSoup):
             "div", class_="table tablemenu"
         )
         self.contact = join(html.find("h5", string="Contact").parent.span.a["href"])
-        self.follow = join(
-            profile_raw.find_all(
-                "h5",
-                string=[
-                    "Mod watch",
-                    "Game watch",
-                    "Group watch",
-                    "Engine watch",
-                    "Hardware watch",
-                    "Software watch",
-                ],
-            )[0].parent.span.a["href"]
-        )
+
+        self.follow = join(html.find("a", title="Follow")["href"])
 
         try:
             share = profile_raw.find("h5", string="Share").parent.span.find_all("a")
@@ -874,8 +863,8 @@ def __init__(self, html: BeautifulSoup):
             )
 
         try:
-            self.follow = join(profile_raw.find("h5", string="Member watch").parent.span.a["href"])
-        except AttributeError:
+            self.follow = join(html.find("a", title="Follow")["href"])
+        except TypeError:
             LOGGER.info(
                 "Can't watch yourself, narcissist...", exc_info=LOGGER.level >= logging.DEBUG
             )
@@ -1073,7 +1062,9 @@ def __init__(self, **kwargs):
     def __repr__(self):
         return f"<Option text={self.text}>"
 
-T = TypeVar('T')
+
+T = TypeVar("T")
+
 
 class ModDBList(collections.abc.MutableSequence[T], Generic[T]):
     """Base List type for the lib

moddb/client.py

@@ -5,6 +5,7 @@
 import sys
 from typing import TYPE_CHECKING, Any, List, Tuple, Union
 
+from curl_adapter import CurlCffiAdapter
 import requests
 from bs4 import BeautifulSoup
 from requests import utils
@@ -20,8 +21,8 @@
     GLOBAL_LIMITER,
     GLOBAL_THROTLE,
     LOGGER,
-    SSLAdapter,
     concat_docs,
+    create_login_payload,
     generate_hash,
     generate_login_cookies,
     get,
@@ -30,6 +31,7 @@
     get_sitearea,
     get_siteareaid,
     join,
+    prepare_request,
     raise_for_status,
     ratelimit,
     soup,
@@ -295,7 +297,8 @@ class Client:
 
     def __init__(self, username: str, password: str):
         session = requests.Session()
-        session.mount("https://", SSLAdapter())
+        session.mount("http://", CurlCffiAdapter())
+        session.mount("https://", CurlCffiAdapter())
         session.cookies = generate_login_cookies(username, password, session=session)
         self._session = session
         LOGGER.info("Login successful for %s", username)
@@ -1175,3 +1178,90 @@ def downvote_tag(self, tag: Tag) -> bool:
             Whether the downvote was successful
         """
         return self._vote_tag(tag, 1)
+
+
+class TwoFactorAuthClient(Client):
+    """A subclass of client to be used when facing 2FA requirements."""
+
+    def __init__(self, username: str, password: str):
+        self.username = username
+        self.password = password
+
+        session = requests.Session()
+        session.mount("http://", CurlCffiAdapter())
+        session.mount("https://", CurlCffiAdapter())
+        self._session = session
+        self.member: Member = None
+
+        self._2fa_request: requests.Response = None
+
+    def __repr__(self):
+        return f"<Client username={self.username}>"
+
+    def login(self) -> bool:
+        """Log the user in
+
+        Returns
+        --------
+        bool
+            True if the login was successful, false it the login requires 2FA
+        """
+        data, resp = create_login_payload(self.username, self.password, self._session)
+
+        req = requests.Request("POST", f"{BASE_URL}/members/login", data=data, cookies=resp.cookies)
+        login = self._session.send(prepare_request(req, self._session), allow_redirects=False)
+
+        if "members2faemailhash" in login.text:
+            self._2fa_request = login
+            return False
+
+        if "freeman" not in login.cookies:
+            raise ValueError(f"Login failed for user {self.username}")
+
+        self._session.cookies = login.cookies
+
+        self.member = Member(
+            soup(self._request("GET", f"{BASE_URL}/members/{self.username.replace('_', '-')}").text)
+        )
+        return True
+
+    def submit_2fa_code(self, code: str) -> Member:
+        """Submit the 2FA code sent to the user being logged in. Only works if called
+        after `login`
+
+        Returns
+        --------
+        Member
+            The logged in member
+        """
+        if self._2fa_request is None:
+            raise ValueError("Call login first")
+
+        html = soup(self._2fa_request.text)
+        form = html.find("form", action="https://www.moddb.com/members/login2fa/#membersform")
+
+        data = {
+            "rememberme": "1",
+            "referer": "/",
+            "2faemaildomain": form.find("input", id="members2faemaildomain")["value"],
+            "2faemailhash": form.find("input", id="members2faemailhash")["value"],
+            "2faemailcode": code,
+            "members": "Verify",
+        }
+
+        req = requests.Request(
+            "POST", f"{BASE_URL}/members/login2fa", data=data, cookies=self._2fa_request.cookies
+        )
+        login = self._session.send(prepare_request(req, self._session), allow_redirects=False)
+
+        if "freeman" not in login.cookies:
+            raise ValueError(f"Login failed for user {self.username}")
+
+        self._session.cookies = login.cookies
+        self._2fa_request = None
+
+        self.member = Member(
+            soup(self._request("GET", f"{BASE_URL}/members/{self.username.replace('_', '-')}").text)
+        )
+
+        return self.member

moddb/errors.py

@@ -20,3 +20,11 @@ def __init__(self, message, remaining) -> None:
         super().__init__(message)
 
         self.remaining = remaining
+
+
+class AuthError(ModdbException):
+    """The user you are trying to login with requires 2FA to login. Use
+    the TwoFactorAuthClient object to do so.
+    """
+
+    pass

moddb/pages/article.py

@@ -129,7 +129,7 @@ def __init__(self, html: bs4.BeautifulSoup):
         self.summary = html.find("p", class_="introductiontext").string
 
         if self.category == ArticleCategory.tutorials:
-            cat = html.find("span", itemprop="proficiencyLevel").nextSibling.strip()
+            cat = html.find("span", itemprop="proficiencyLevel").next_sibling.strip()
             self.tutorial_category = TutorialCategory[
                 cat.replace("/", "_").replace(" ", "_").lower()
             ]

moddb/pages/base.py

@@ -59,17 +59,19 @@ def __init__(self, html: BeautifulSoup):
                 ),
                 lambda: int(html.find("input", attrs={"name": "siteareaid"})["value"]),
                 lambda: int(html.find("meta", property="og:image")["content"].split("/")[-2]),
-                lambda: re.match(
+                lambda: re.findall(
                     r"https:\/\/www\.moddb\.com\/html\/scripts\/autocomplete\.php\?a=mentions&p=home&l=6&u=(\d*)",
                     str(html),
-                ).group(1),
+                )[0],
             ]
         ):
             try:
                 self.id = func()
                 break
-            except (AttributeError, TypeError):
-                LOGGER.warning("Failed to get id from method %s for member %s", index, self.name)
+            except (AttributeError, TypeError) as e:
+                LOGGER.warning(
+                    "Failed to get id from method %s for member %s: %s", index, self.name, e
+                )
         else:
             raise AttributeError(f"Failed to get id from member {self.name}")