ptt-crawler/scraptt/spiders/utils/parsers/html/utils.py

from typing import List
from scrapy.selector import Selector
from scrapy.http.response.html import HtmlResponse


def get_title_tags(response: HtmlResponse) -> List[Selector]:
    """The get_title_tags function gets the title tags DOM.
    Args:
        response (HtmlResponse): the scrapy HtmlResponse.
    Returns:
        a list of scrapy anchor tag selectors
    """
    title_css = ".r-ent .title a"

    if response.url.endswith("index.html"):
        return response.dom(".r-list-sep").prev_all(title_css)

    return response.dom(title_css)