"""
For consistency the relative paths in this module shall never have ambiguous values like ".".
"""

import asyncio
import errno
import logging
import os
import re
import shutil
import stat
import time
from collections import deque
from contextlib import asynccontextmanager
from pathlib import Path
from typing import List, Dict, Optional, Tuple, Union, Set, Callable, Awaitable, Deque

import aiohttp
import pybase64
import wait_for2

from clarabot_nano.common.solr.doc import SolrRequestFailed, query_iter_docs, decompose_docs_by_parent_accessor
from clarabot_nano.common.solr.rest import (
    nano_update_post,
    preparse_solr_response,
    get_solr_error,
    serialize_solr_update_json,
)
from clarabot_nano.common.utility import IncreasingTimeout
from clarabot_nano.indexer.commit_tracker import DocumentCommitTracker
from clarabot_nano.indexer.config import IndexerConfig
from clarabot_nano.indexer.event import SchedulerFilesystemEvent
from clarabot_nano.indexer.extractor.base import (
    ExtractorInoperable,
    RetryExtractError,
    ServerInternalExtractError,
    AbstractExtractionSource,
    AbstractContentExtractor,
)
from clarabot_nano.indexer.extractor.picoture import PicotureContentExtractor, PicotureExtraction
from clarabot_nano.indexer.extractor.source import FileExtractionSource
from clarabot_nano.indexer.extractor.tika import TikaContentExtractor, TikaExtraction
from clarabot_nano.indexer.idxstatedb import IndexingStateDatabase
from clarabot_nano.indexer.language import LanguageDetector
from clarabot_nano.indexer.observer import (
    Observer,
    FileSystemEvent,
    FileSystemMovedEvent,
    EVENT_TYPE_DELETED,
    EVENT_TYPE_CREATED,
    EVENT_TYPE_MODIFIED,
)
from clarabot_nano.indexer.utility import (
    create_docs_query_children,
    create_accessor_path,
    create_docs_query_exact,
    create_doc_query_path,
    convert_accessor_to_path,
    convert_accessor_to_parent_accessor,
)
from clarabot_nano.utility.aio import run_tasks
from clarabot_nano.utility.path.chroot import (
    ChrootPath,
    ResolvedRootConsistencyError,
    PartialNodeResolutionError,
    CycledPathError,
)
from clarabot_nano.utility.path.common import ConditionalValidationError
from clarabot_nano.utility.path.error import coerce_platform_os_error, BroadNotFoundError, NtOSError
from clarabot_nano.utility.path.validate import ValidationError, normalize_absolute_path

logger = logging.getLogger(__name__)


class SyncInterrupted(Exception):
    pass


class SyncFilesystemItem(object):
    __slots__ = ("name", "path", "stat", "accessor_path", "posix_path")

    def __init__(self, name: str, path: Path, s: os.stat_result, accessor_path: str):
        # Name that is used in the accessor path. It is the name of the symlink if the item is a symlink for example.
        self.name = name
        # Path is resolved. If item is a symlink this path may not be the child of the parent directory from listing.
        self.path = path
        # The path's lstat.
        self.stat = s
        # The accessor paths are pre-generated for checking commit state.
        self.accessor_path = accessor_path
        # Posix representation of the path.
        self.posix_path = self.path.as_posix()

    def create_extraction_source(self, session: aiohttp.ClientSession, ec: IndexerConfig) -> FileExtractionSource:
        return FileExtractionSource(session, ec, self.name, self.path, self.stat)


class SyncFilesystemParent(object):
    """
    Collect and prepare items in a directory for synchronization.
    """

    __slots__ = ("root_path", "rel", "path", "accessor_path", "posix_path", "items")

    def __init__(self, root_path: str, rel: str, path: Path):
        # Root is resolved from a chroot.
        self.root_path = root_path
        # The relative path that this object is listing the children of.
        self.rel = rel
        # Resolved path of root+rel that is a directory.
        self.path = path
        # Accessor path of the parent/directory.
        self.accessor_path = create_accessor_path(self.root_path, self.rel)
        # Posix representation of the path.
        self.posix_path = self.path.as_posix()
        # NOTE: could use SortedList but memory conservation seems more beneficial
        self.items: List[SyncFilesystemItem] = []

    def add_item(self, name: str, path: Path, s: os.stat_result) -> SyncFilesystemItem:
        self.items.append(
            item := SyncFilesystemItem(name, path, s, create_accessor_path(self.root_path, self.rel, name))
        )
        return item

    def create_parent_directory_update(self):
        """
        Create parent document update, for server cache invalidation. The "mtime" field is repurposed and will hold
        the last timestamp that a direct child document was updated.
        NOTE: The commit tracker is not and needs not to be aware of this additional update to the parent doc. Any
            child document of a parent is only updated after the parent is checked to exist. While the parent of any
            document is only updated by the update of their children.
        NOTE: This may insert a directory item for the drive-root, which is omitted otherwise.
        """
        return {
            "path": self.posix_path,
            "file": False,
            "accessors": {"add-distinct": self.accessor_path},
            # Having a nanosecond timestamp is sufficient for reliable invalidation. The time that shall pass between
            # each file update is greater by orders of magnitude than the resolution. Even if the system time is
            # adjusted backwards, having an update match the time of the previous one is highly unlikely.
            "mtime": time.time_ns(),
        }


class ObserverEventHandler(object):
    """
    Decompose the events from the processing thread to the indexer's event queue.
    """

    def __init__(self, loop: asyncio.AbstractEventLoop, event_queue: asyncio.Queue):
        self._loop = loop
        self._event_queue = event_queue

    def _queue_event(self, event: SchedulerFilesystemEvent):
        try:
            self._event_queue.put_nowait(event)
        except asyncio.QueueFull:
            logger.warning("Filesystem sync event dropped because queue is full")

    def dispatch(self, event: FileSystemEvent):
        try:
            if isinstance(event, FileSystemMovedEvent):
                self._loop.call_soon_threadsafe(
                    self._queue_event, SchedulerFilesystemEvent(src := normalize_absolute_path(event.src_path), False)
                )
                self._loop.call_soon_threadsafe(
                    self._queue_event, SchedulerFilesystemEvent(normalize_absolute_path(event.dest_path), src)
                )
            elif event.event_type == EVENT_TYPE_DELETED:
                self._loop.call_soon_threadsafe(
                    self._queue_event, SchedulerFilesystemEvent(normalize_absolute_path(event.src_path), False)
                )
            elif event.event_type == EVENT_TYPE_CREATED:
                self._loop.call_soon_threadsafe(
                    self._queue_event, SchedulerFilesystemEvent(normalize_absolute_path(event.src_path), True)
                )
            elif event.event_type == EVENT_TYPE_MODIFIED and not event.is_directory:
                # directory modifications are ignored; those are emitted when updating their timestamps?
                self._loop.call_soon_threadsafe(
                    self._queue_event, SchedulerFilesystemEvent(normalize_absolute_path(event.src_path), True)
                )
        except (OSError, ValidationError):
            logger.warning("Filesystem sync event dropped because path is not absolute", exc_info=True)


class IndexingSchedulerEngine(object):
    """
    To simplify the Solr index consistency guarantees by avoiding parallel updates or intermixed queries and updates,
    the synchronization will support interruption for higher priority events to be processed. Serial performance is
    desired to surely restrict resource consumption.

    Execution tree of the synchronization:
        - traversal iterators (interrupt handling and priority queues)
            - _sync_path
                - _drop_document_tree
                - _lose_document_tree
                - _sync_file
                    - _get_docs
                    - _sync_execute
                        - _drop_document_tree
                        - _add_file
                        - _lose_document_tree
                - _sync_dir
                    - _get_docs
                    - _sync_execute
                        - _drop_document_tree
                        - _add_file
                        - _lose_document_tree

    Because the synchronization relies heavily on the committed state of the indexed documents' accessors, it is
    unfortunately required for us to commit/soft-commit frequently. After each update in worst case scenario.
    To avoid having to explicitly commit we explored if the data in question could be stored in the index-state-db.
    While it is not out of the question, its benefits would come with the double storage-drawback because all fields
    would still need to be stored in solr for partial updates. Unless we lose the "de-duplicating" quality of the
    schema for nested *and* multi named (symlinked) files, partial updates will be required.
    If all documents would be indexed as-is for each of their respective roots and for all of their alternate names
    individually, there would be no reason to have a lot of fields stored or support partial updates. The only extra
    data storage would come from the explicit alternate names and the different roots. The commits in this case would
    still be needed, but they could be optimized better than before.
    Either way soft-commit is required if there are relevant tlog docs in flight and we're querying docs under a parent
    for their children or recursively:
        - start by an explicit soft-commit
        - track changes to docs by parent with a ttldict
            - consult this before a query and execute a soft-commit if necessary

    """

    DOC_SLICE_ITER_DIR = 300
    DOC_SLICE_DROP_TREE = 50

    SYNC_INTERRUPT_CONTEXT_BOTH = 1  # fs and indexed entities both exist
    SYNC_INTERRUPT_CONTEXT_INSERT = 2  # fs items are being inserted into index
    SYNC_INTERRUPT_CONTEXT_REMOVE = 3  # index items no longer exist on fs
    SYNC_INTERRUPT_CONTEXT_DELETE = 4  # recursive document tree removal

    FAILED_ROOTS_CHECK_INTERVAL = 15.0

    IMPLICIT_SYNC_RETRY_ATTEMPTS = 2
    IMPLICIT_EXTRACT_RETRY_ATTEMPTS = 2

    INCREASING_TIMEOUT = IncreasingTimeout(1.0, 15.0, 0.2)

    # Names that are temporary or are a technical detail of some feature should not be indexed at all.
    IGNORE_TEMPORARY_NAME_RE = re.compile(r"^\.~CbN\..+#$", re.UNICODE)

    def __init__(
        self,
        session: aiohttp.ClientSession,
        ec: IndexerConfig,
        index_state_db: IndexingStateDatabase,
        core_name: str,
        languages: List[str],
        core_config: dict,
        event_queue: asyncio.Queue,
        set_status: Callable[[str], Awaitable],
        notify_work: Callable[[str], None],
    ):
        assert self.DOC_SLICE_ITER_DIR > 1
        assert self.DOC_SLICE_DROP_TREE > 1
        self._loop = asyncio.get_event_loop()
        self.session = session
        self.ec = ec
        self.index_state_db = index_state_db
        self.core_name = core_name
        self.languages = set(languages)
        self._template_document = {"summary_faba": None, "text_faba": None}
        for lang in self.languages:
            self._template_document[f"summary_{lang}"] = None
            self._template_document[f"text_{lang}"] = None

        self._set_status = set_status
        self._notify_work = notify_work

        self._lang_det = LanguageDetector()
        self._tika_extractor = TikaContentExtractor(self.session, self.ec)
        self._picoture_extractor = PicotureContentExtractor(self.session, self.ec)

        self._sync_is_polling = False
        self._high_priority_sync_active = False
        self._sync_lock = asyncio.Lock()  # to serialize low and high priority synchronization tasks
        self._solr_has_free_space = asyncio.Event()  # to halt any processing
        self._priority_traversal_added = asyncio.Event()  # to wake the high priority processing
        self._event_queue = event_queue
        self._event_handler = ObserverEventHandler(self._loop, self._event_queue)
        self._last_path_sync_done_t = 0.0

        # Event aggregation to conserve resources.
        self._event_aggregate_wait = 0.8  # time to wait for an event to have a matching one arrive
        self._event_aggregate_bucket_time = 0.2  # time resolution to start new buckets by
        self._event_aggregate_buckets: Deque[Tuple[float, Set[Tuple[str, str]]]] = deque()
        self._event_aggregate_new_event = asyncio.Event()  # idle waiter event for the aggregate processing task

        # Track in-flight transactions that may have not been committed yet. By managing this state (and having a
        # single solr-managing process), we can optimize soft-commit calls to be omitted or only be requested when
        # necessary.
        try:
            auto_soft_commit_max_time = max(1.5, core_config["updateHandler"]["autoSoftCommit"]["maxTime"] / 2000.0)
        except (KeyError, TypeError, ValueError):
            auto_soft_commit_max_time = 8.0
        self._commit_tracker = DocumentCommitTracker(auto_soft_commit_max_time)

    async def run(self):
        # Check if extractors are ready.
        await self._tika_extractor.ensure_operable()
        await self._picoture_extractor.ensure_operable()
        if self.ec.indexer_force_sync_on_startup:
            # Re-start the polling sync on all roots at starting to quickly find changes made when the indexer
            # was shut down. This can be disabled by config if the owner wants.
            await self.index_state_db.reset_poll_traversal()
        solr_error_count = 0
        solr_error_time = 0.0
        while True:
            try:
                chroots, failed_roots = await self._prepare_synchronization()
                tasks = [
                    asyncio.ensure_future(self._monitor_index_free_space()),
                    asyncio.ensure_future(self._poll_sync_traverse(chroots)),
                    asyncio.ensure_future(self._priority_sync_traverse(chroots)),
                    asyncio.ensure_future(self._observe_drives(chroots)),
                    asyncio.ensure_future(self._process_sync_events(chroots)),
                    asyncio.ensure_future(self._process_event_aggregation_buckets()),
                    asyncio.ensure_future(self._auto_soft_commit()),
                    asyncio.ensure_future(self.index_state_db.closed.wait()),
                ]
                if failed_roots:
                    tasks.append(asyncio.ensure_future(self._observe_failed_roots(chroots, set(failed_roots))))
                await run_tasks(tasks)
                break
            except ResolvedRootConsistencyError:
                pass  # restart processes with adjusted chroots
            except ExtractorInoperable:
                logger.error("Waiting for extractor after it became inoperable")
                await asyncio.gather(
                    self._tika_extractor.ensure_operable(timeout=30.0),
                    self._picoture_extractor.ensure_operable(timeout=30.0),
                )  # wait more for the extractors to recover
            except SolrRequestFailed:
                if time.perf_counter() - solr_error_time >= 120.0:  # reset count if last error was a while ago
                    solr_error_count = 0
                solr_error_count += 1
                solr_error_time = time.perf_counter()
                if solr_error_count > 10:
                    raise
                logger.warning("Sync stopped by Solr error, retrying...", exc_info=True)
                await asyncio.sleep(self.INCREASING_TIMEOUT.get(solr_error_count))

    async def _prepare_synchronization(self) -> Tuple[Dict[str, ChrootPath], List[str]]:
        """
        This runs prior to parallelization of tasks, so there won't be any concurrent issues.
        """
        chroots, failed_roots = self.ec.get_drive_chroots()
        # The state db manages the resolved roots that are indexed.
        old_roots = await self.index_state_db.update_roots(set(chroots.keys()))
        # If there are any failed roots, we cannot discern if an old root is by removed configuration or temporarily
        # unavailable storage. Depending on having failed roots initiate immediate or lose-delayed removal.
        for old_root in old_roots:
            if not failed_roots or await self._lose_document_tree(old_root, ""):
                await self._solr_soft_commit()  # removals need serializable consistency
                await self._drop_document_tree(create_accessor_path(old_root))
                await self.index_state_db.delete_root(old_root)
        # start synchronization by ensuring everything is visible to the searchers
        await self._solr_soft_commit()
        return chroots, failed_roots

    async def _monitor_index_free_space(self):
        """
        The size of the solr index can grow considerably, so monitoring free space there is necessary.
        We can only prevent storage filling by ourselves by estimating maximum write size and periodically checking.
        Other processes may quickly fill up the disk that we cannot track.
        The solr data dir should always be available, so errors are not well defined or handled.
        """
        while True:
            try:
                du = await self._loop.run_in_executor(None, shutil.disk_usage, self.ec.solr_data_dir)
            except OSError:
                logger.exception("Failed to check disk usage for solr data directory")
                wait = 60
            else:
                if du.free > self.ec.indexer_reserved_space:
                    self._solr_has_free_space.set()
                    wait = 150
                else:
                    self._solr_has_free_space.clear()
                    wait = 30
            await asyncio.sleep(wait)

    async def _observe_drives(self, chroots: Dict[str, ChrootPath]):
        """"""
        if not chroots or Observer is None:
            # Wait for config change, don't do anything without a single root or without an available observer.
            while True:
                await asyncio.sleep(15)
        observer = Observer()
        for path in chroots.keys():  # normalized, validated and resolved posix; only convert path to platform seps
            observer.schedule(self._event_handler, str(Path(path)), recursive=True)
        observer.start()
        try:
            await self._observe_drives_loop(observer)
        finally:
            observer.stop()
            t = time.perf_counter() + 10
            while time.perf_counter() < t:
                observer.join(0.5)
                if not observer.is_alive():
                    break
                await asyncio.sleep(0.5)
            else:
                raise Exception("Drive observers did not exit within time limit")

    async def _observe_drives_loop(self, observer: Observer):
        """
        This is made to be a method, so testing can wait for the FS-event observing task before starting.
        """
        while True:
            await asyncio.sleep(60.0)

    async def _process_sync_events(self, chroots: Dict[str, ChrootPath]):
        """
        Process events collected by filesystem observers or submitted by other nano processes.
        If an event (path) matches for a valid chroot, a sync item will be put into the state db.
        """
        while True:
            event: SchedulerFilesystemEvent = await self._event_queue.get()
            try:
                ready = event.is_ready()
            except OSError:
                # Errors here can be temporary for various reasons. The storage or the system may be busy and it
                # may return an error, rather than wait for the api-call itself. So we're waiting on it to see if it
                # resolves or not.
                if event.is_error_stalled(inc=True, limit=24):
                    logger.warning("Abandoning stalled event by errors for %s", event.path, exc_info=True)
                    continue
                ready = False
            if not ready:
                self._loop.call_later(1.0, self._delay_queue_event, event)
                continue
            # The event's path is normalised, but only validated if it is an arbitrary event from an other Nano
            # process. Observed events come from the OS "directly", so they must be valid.
            path = event.path.as_posix()  # normalize to posix path separators
            traversal_items: List[Tuple[str, str]] = []
            for root in chroots.keys():
                if path.startswith(root):
                    if (rl := len(root)) == len(path):
                        traversal_items.append((root, ""))
                    elif path[rl] == "/":
                        traversal_items.append((root, path[rl + 1 :]))
            if traversal_items:
                # There are several cases when a file is continuously changed. In those cases frequently
                # re-indexing them should generally be avoided to conserve resources. For this reason a bucketed
                # queue of changes is used to aggregate frequent events.
                t = time.perf_counter() + self._event_aggregate_wait
                if self._event_aggregate_buckets:
                    for bucket in self._event_aggregate_buckets:
                        bucket[1].difference_update(traversal_items)
                    if t - self._event_aggregate_buckets[-1][0] > self._event_aggregate_bucket_time:
                        self._event_aggregate_buckets.append((t, set(traversal_items)))
                    else:
                        self._event_aggregate_buckets[-1][1].update(traversal_items)
                else:
                    self._event_aggregate_buckets.append((t, set(traversal_items)))
                self._event_aggregate_new_event.set()

    async def _process_event_aggregation_buckets(self):
        """
        Manage the event aggregation buckets that are expiring their max delay.
        """
        while True:
            # idle waiter using an event
            await self._event_aggregate_new_event.wait()
            self._event_aggregate_new_event.clear()
            # inner work loop
            while self._event_aggregate_buckets:
                process_after = self._event_aggregate_buckets[0][0]
                process_after -= time.perf_counter()  # turn timestamp into delay
                if process_after > 0:
                    await asyncio.sleep(process_after)
                traversal_items = self._event_aggregate_buckets.popleft()[1]
                # Add the new sync items to the state db then notify the processing task.
                await self.index_state_db.add_priority_traversal_items(traversal_items)
                self._priority_traversal_added.set()

    def _delay_queue_event(self, event: SchedulerFilesystemEvent):
        try:
            self._event_queue.put_nowait(event)
        except asyncio.QueueFull:
            logger.warning("Delayed event dropped because queue is full")

    async def _auto_soft_commit(self):
        """
        Do "frequent" auto soft commits to solr, so we can have selective serial consistency where we want.
        """
        while True:
            time_until_commit = self._commit_tracker.get_time_until_commit()
            if time_until_commit is None:
                time_until_commit = 60.0  # No documents are queued for commit, wait for something to happen.
            elif time_until_commit <= 0.0:
                # It's time to do a commit. There might be synchronizations running right now, which will also do
                # the commit when they get to a check-point. Commits must be serialized, so wait for _sync_lock.
                async with self._sync_lock:
                    # The commit could have been done by an other task before we acquired the lock, so re-check it
                    # to avoid superfluous commits.
                    time_until_commit = self._commit_tracker.get_time_until_commit()
                    if time_until_commit is None:
                        time_until_commit = 60.0  # commit done by other task
                    elif time_until_commit <= 0.0:
                        await self._solr_soft_commit()
                        time_until_commit = 60.0
            try:
                await wait_for2.wait_for(self._commit_tracker.new_time.wait(), timeout=time_until_commit)
            except asyncio.TimeoutError:
                pass

    async def _observe_failed_roots(self, chroots: Dict[str, ChrootPath], root_paths: Set[str]):
        """
        New or different roots from configurations are handled by the worker process. Usable roots when changed will
        raise ResolvedRootConsistencyError during use by a synchronization. Roots that were unsuccessfully initialised
        need to be checked periodically to see when they become available.
        """
        while root_paths:
            await asyncio.sleep(self.FAILED_ROOTS_CHECK_INTERVAL)
            for root_path in list(root_paths):
                try:
                    resolved_root = ChrootPath(
                        root_path, restrict_sensitive=self.ec.restrict_sensitive_path_access
                    ).root.as_posix()
                except (OSError, ValidationError):
                    continue
                if resolved_root not in chroots:
                    raise ResolvedRootConsistencyError("Failed root has become accessible")
                else:
                    chroots[resolved_root].add_alternate_root_path(root_path)
                root_paths.remove(root_path)  # failed root is now resolved
        # All failed roots have been found, but none have raised consistency error. They've been added to already
        # existing chroots, so further consistency errors will be raised from sync instead.
        while True:
            await asyncio.sleep(600.0)

    def _handle_processing_interrupt(self, context: int):
        """
        Raise SyncInterrupted if there is an active condition that indicates current tasks should be halted.
        Only call this from contexts where the _sync_lock is held.
        """
        if self._sync_is_polling and self._high_priority_sync_active:
            raise SyncInterrupted()

    async def _poll_sync_traverse(self, chroots: Dict[str, ChrootPath]):
        """
        Periodic traversal of all roots with low priority.
        """
        if not chroots:
            await self._set_status("ready")  # idle
            # Wait for config change, don't do anything without a single root.
            while True:
                await asyncio.sleep(15)
        await self._set_status("scanning")
        # Initialise the poll-begin timestamp if it is not present.
        await self.index_state_db.init_poll_traversal()
        while True:
            # Get the last root whose traversal shall be continued.
            ltr = await self.index_state_db.get_last_poll_traversal_root()
            if chroot := chroots.get(ltr):
                while True:
                    # Get next path to traverse.
                    ntp = await self.index_state_db.next_poll_traversal_path()
                    if ntp is None:
                        break
                    # Execute synchronization on the traversal-path, handle high-priority interrupt and resumption.
                    while True:
                        await self._solr_has_free_space.wait()
                        try:
                            async with self._sync_lock:
                                try:
                                    self._sync_is_polling = True
                                    async with self._sync_path_delay_track_done_time():
                                        next_paths = await self._sync_path(chroot, ntp)
                                finally:
                                    self._sync_is_polling = False
                            break
                        except SyncInterrupted:
                            pass
                    # Clear current path of traversal and queue children to be synchronized.
                    await self.index_state_db.update_poll_traversal_paths(ntp, next_paths)
                # Set the polled time of the root, so it is no longer eligible for processing in the current cycle.
                await self.index_state_db.finish_poll_traversal_root(ltr)
            # Select a new root for traversal because ltr is done or invalid.
            if not (roots_to_process := await self.index_state_db.get_poll_traversal_roots_to_process()):
                root = None
            else:
                for root in chroots.keys():
                    if root in roots_to_process and (not ltr or root != ltr):
                        break  # process root for next
                else:
                    root = None
            if root:
                # Start processing the selected root. This will reset the paths buffer and set the ltr.
                await self.index_state_db.start_poll_traversal_root(root)
            else:
                # All roots have been traversed, the finish_poll_traversal() will clear obsolete lost records and the
                # ltr. The first call after each start_poll_traversal() will set the finish time for delay-control.
                time_since_traversal_finish = await self.index_state_db.finish_poll_traversal(list(chroots.keys()))
                await self._set_status("ready")
                await self._poll_sync_finish_delay(time_since_traversal_finish)
                # Increment the poll begin time, so all roots become eligible for traversal again.
                await self.index_state_db.start_poll_traversal()
                await self._set_status("scanning")

    async def _poll_sync_finish_delay(self, time_since_traversal_finish: float):
        """
        Handle polling synchronization finish. The time argument is zero if the traversal has just finished. If
        the polling has finished before and this is just a delay until the next cycle then the time argument is
        greater than zero.
        """
        await asyncio.sleep(self.ec.indexer_delay - time_since_traversal_finish)

    async def _priority_sync_traverse(self, chroots: Dict[str, ChrootPath]):
        """
        High priority traversal of paths received from various events.
        The events are collected in the state db because the deep traversal could grow the horizon of sync-targets
        considerably.
        """
        while True:
            item: Optional[Tuple[str, str, int]] = await self.index_state_db.next_priority_traversal_item()
            if item:
                try:
                    self._high_priority_sync_active = True
                    async with self._sync_lock:
                        while True:
                            await self._solr_has_free_space.wait()
                            root, path, merge = item
                            if chroot := chroots.get(root):
                                while True:
                                    try:
                                        async with self._sync_path_delay_track_done_time():
                                            next_paths = await self._sync_path(chroot, path)
                                        break
                                    except SyncInterrupted:
                                        pass  # May be raise when a file is no longer available as it had been listed.
                            else:
                                next_paths = []  # Just skip syncing what is not under valid roots.
                            await self.index_state_db.update_priority_traversal_item(root, path, merge, next_paths)
                            item = await self.index_state_db.next_priority_traversal_item()
                            if not item:
                                break
                finally:
                    self._high_priority_sync_active = False
            await self._priority_traversal_added.wait()
            self._priority_traversal_added.clear()

    @asynccontextmanager
    async def _sync_path_delay_track_done_time(self):
        """
        Basic throttling support.
        Priority traversal items may be aggregated and thus optimized by waiting here for frequently changing files.
        """
        delay = self.ec.indexer_minimum_delay_between_path_sync
        if delay > 0.0:  # config
            delay -= time.perf_counter() - self._last_path_sync_done_t
            if delay > 0.0:  # timing
                await asyncio.sleep(delay)
        try:
            yield
        finally:
            self._last_path_sync_done_t = time.perf_counter()

    async def _sync_path(self, chroot: ChrootPath, rel: str) -> Optional[List[str]]:
        """
        Called by polling and priority traversals to update the documents by the supplied path.
        If the path is a directory the children directories will be returned to help the traversal move along.
        If the path is anything else the return value is omitted.
        This is the entry point to all synchronization procedures and the _sync_lock must be acquired beforehand.
        Wrap this with _sync_path_delay_track_done_time() to throttle processing.
        TODO: add a timeout on this for safety against stucking?
        """
        assert self._sync_lock.locked()
        attempt = 0  # implicit retry counter
        root_path = chroot.root.as_posix()
        listing_success = True  # constant; immediate removals are handled here and not passed to _sync_dir
        while True:
            try:
                with coerce_platform_os_error():  # Collapse platform errors to specifically handled ones.
                    try:
                        paths, s, dir_items = chroot.node(rel)
                    except PartialNodeResolutionError as e:
                        # Path was found to be a directory... but it may have already changed. (causing the error)
                        paths, s, e = e.args
                        if isinstance(e, (FileNotFoundError, NotADirectoryError)):
                            # NOTE: The original error can be a platform specific one and avoid this branch. This
                            # is not a problem, because if the problem persists implicit-retry exhaustion will
                            # trigger losing the path, otherwise it will be handled as a new error or success is
                            # received.
                            attempt += 1
                            if attempt <= self.IMPLICIT_SYNC_RETRY_ATTEMPTS:
                                await asyncio.sleep(0.15)  # implicit retry delay so the FS may recover
                                continue  # retry resolution
                            raise e
                        else:  # NOTE: e can't be ValidationError here
                            raise e
            except (ResolvedRootConsistencyError, ExtractorInoperable):
                raise  # Handled at traversal or work-coroutine.
            except (
                # Symlinks have looped. Cycles are forbidden in indexing.
                CycledPathError,
                # Failure by some policy violation. Some symlinks point to inappropriate places most likely.
                ConditionalValidationError,
                # Failure by filesystem operation. The path is obsolete in some way.
                FileNotFoundError,
                NotADirectoryError,
                # Failure by filesystem operation. Permissions have been revoked to list a directory or read a file.
                PermissionError,
            ) as e:
                # The path resolution has failed so all matching docs shall be removed from the index. (normally)
                # It is not clear at what point the resolution of path has an error. We assume most of the time
                # the error will be at the final part. This shall cover the common cases of errors. In case the
                # error is further up the path the next sync or other event-triggered syncs will fix those documents.
                # For symlink cycling the triggering item (directory) can be kept. It is added by the parent traversal
                # so the child sync should not remove it.
                parent_accessor_path, fs = await self._resolve_sync_error_parent(chroot, rel, root_path)
                await self._solr_check_commit(parent_accessor_path)
                await self._drop_document_tree(
                    create_accessor_path(root_path, rel), keep_exact=isinstance(e, CycledPathError), fs=fs
                )
                return
            except ValidationError:
                # The path to be processed has encountered an error that likely cannot be resolved by retrying or
                # config or policy change. Because such errors should not usually happen, it is hard to say how to
                # handle them best. For now, we log a warning with the exception info and run a general cleanup IF
                # rel is not empty (which it should not be).
                logger.warning("Path for indexing has failed validation: %s %s", root_path, rel, exc_info=True)
                if rel:
                    parent_accessor_path, fs = await self._resolve_sync_error_parent(chroot, rel, root_path)
                    await self._solr_check_commit(parent_accessor_path)
                    await self._drop_document_tree(create_accessor_path(root_path, rel), fs=fs)
                return
            except Exception as e:
                # We need to heuristically decide how to proceed with an unknown or generic error. This could be a
                # temporary issue - like a NFS having some problem - or some policy permanently denying the access to
                # some FS item, but failing to give a good reason for it either by design or the lack of it. The
                # sync operation must not stall permanently or even for a long time. Raising here would reset the
                # indexing process, but it eventually results in a retry. If the target was indexed previously, we
                # need to remove the docs if an error persists here.
                logger.exception("Path for indexing has failed unexpectedly (%s): %s %s", attempt, root_path, rel)
                if not isinstance(e, BroadNotFoundError):
                    attempt += 1
                    if attempt <= self.IMPLICIT_SYNC_RETRY_ATTEMPTS:
                        await asyncio.sleep(0.15)  # implicit retry delay so the FS may recover
                        continue  # retry resolution
                # The path is not accessible for some reason. Consider it lost and eventually remove it.
                if not await self._lose_document_tree(root_path, rel):
                    return  # delay removal of items until lose conditions are fulfilled
                parent_accessor_path, fs = await self._resolve_sync_error_parent(chroot, rel, root_path)
                await self._solr_check_commit(parent_accessor_path)
                await self._drop_document_tree(create_accessor_path(root_path, rel), fs=fs)
                return
            # check if the final path's rel is the rel we requested.
            if paths[-1][1] != rel:
                logger.error("Inconsistent relative resolution for %s", rel)
                return
            # Resolution successful.
            if dir_items is None:  # final path is a file
                # The path is implied to be not the root itself, because that is ensured to be a directory.
                # This means 'rel' has to be some valid path that ultimately does not resolve to the root itself.
                await self._sync_file(root_path, paths, s)
                return
            else:
                child_names = await self._sync_dir(root_path, paths, dir_items, listing_success)
                if rel:
                    return [f"{rel}/{child_name}" for child_name in child_names]
                return child_names

    async def _resolve_sync_error_parent(
        self, chroot: ChrootPath, rel: str, root_path: str
    ) -> Tuple[str, Optional[SyncFilesystemParent]]:
        """
        When there is an error for sync-path resolution, the tree to be dropped may update the parent document of the
        relative path. However, this may only be done if the parent is actually a resolved directory and properly
        set-up in the index, which is what this method ensures.
        In case the parent-sync object is not usable by any error, the parent-update will be skipped.
        """
        parent_rel = rel.rpartition("/")[0]
        try:
            paths, s = chroot.paths_stat(parent_rel)
            if stat.S_ISDIR(s.st_mode):
                fs = await self._create_sync_parent(root_path, paths)
                return fs.accessor_path, fs
        except Exception:
            pass
        return create_accessor_path(root_path, parent_rel), None

    async def _sync_dir(
        self,
        root_path: str,
        paths: List[Tuple[Path, str]],
        dir_items: List[Tuple[Path, Optional[Path]]],
        fs_listed: bool,
    ) -> List[str]:
        """
        Prepare the filesystem listed item data by supplying stats of them. Also query and prepare the indexed
        documents by decomposing them into a list by their accessors. The children directories are selected and
        returned from the fs items to help traversal.
        """
        # preprocess the fs items for sync
        child_dirs = []
        fs = await self._create_sync_parent(root_path, paths)
        child_stat_failed = False
        for dir_child, resolved_child in dir_items:
            child_path = dir_child if resolved_child is None else resolved_child
            try:
                s = child_path.lstat()
            except FileNotFoundError:
                # A listed file can be abruptly removed/moved/replaced. Ignoring this seems to be beneficial as
                # opposed to the whole synchronization stopping.
                child_stat_failed = True
                continue
            if stat.S_ISLNK(s.st_mode):
                # Link resolutions have already been done by the chroot. If this yields a symlink then something has
                # changed and the directory listing must be either re-listed by chroot or this child ignored.
                continue
            # Collect the names of child directory objects for further traversal.
            if stat.S_ISDIR(s.st_mode):
                child_dirs.append(dir_child.name)
            fs.add_item(dir_child.name, child_path, s)
        # If any child stats have failed it could be that the current directory has become inaccessible completely.
        # In that case abort the sync altogether.
        if child_stat_failed:
            s = paths[-1][0].lstat()  # may raise FileNotFoundError
            if stat.S_ISLNK(s.st_mode) or not stat.S_ISDIR(s.st_mode):
                raise NotADirectoryError()
        # preprocess the solr docs for sync
        docs = list(
            decompose_docs_by_parent_accessor(
                fs.accessor_path, await self._get_docs(create_docs_query_children(fs.accessor_path))
            )
        )
        await self._sync_execute(fs, fs_listed, docs)
        return child_dirs

    async def _sync_file(self, root_path: str, paths: List[Tuple[Path, str]], s: os.stat_result):
        """
        This is called if the path to be visited is not a directory, which usually happens when the indexer is
        processing events from the observers or the servers. This compiles the supplied information so a partial
        directory sync will be called as implementation.
        """
        assert len(paths) > 1  # there must be a relative path to a file, since the root can't be one
        # reduce the file sync to partial directory sync
        parent_rel, _, fs_name = paths[-1][1].rpartition("/")
        assert parent_rel == paths[-2][1]
        fs = await self._create_sync_parent(root_path, paths, -2)
        item = fs.add_item(fs_name, paths[-1][0], s)
        raw_docs = await self._get_docs(create_docs_query_exact(fs.accessor_path, item.name))
        # NOTE: filter for the true exact matches; see create_docs_query_exact() for more info
        raw_docs = [doc for doc in raw_docs if item.accessor_path in doc["accessors"]]
        # Usually there will be at most one raw document for the exact query. More documents mean there is a
        # synchronization inconsistency: the same path refers to multiple files. This issue should never arise,
        # but in case it does, the excess documents will be removed by the _sync_execute() below.
        if len(raw_docs) > 1:
            logger.warning("Inconsistent accessor reference: %s %s\n  %s", fs.accessor_path, item.name, len(raw_docs))
        docs = list(decompose_docs_by_parent_accessor(fs.accessor_path, raw_docs))
        # Filter out alternative references to the document. Only the accessor in 'paths' shall be updated.
        docs = [name_doc for name_doc in docs if name_doc[0] == item.name]
        assert [d for _, d in docs] == raw_docs, f"Missing accessor by name:\n {raw_docs}\n {docs}"
        await self._sync_execute(fs, True, docs)

    async def _create_sync_parent(
        self, root_path: str, paths: List[Tuple[Path, str]], p_idx: int = -1
    ) -> SyncFilesystemParent:
        """
        Ensure the parent directories of the sync exist and have no conflicting changes.
        """
        fs = SyncFilesystemParent(root_path, paths[p_idx][1], paths[p_idx][0])
        # Check for in-flight (transacted, but not committed) documents that we will use in this sync round. Normally
        # the parent path of all items are enough to be checked. For additional redundant processing optimization, the
        # symlinks' real path may be checked additionally.
        await self._solr_check_commit(fs.accessor_path)
        if not fs.rel:
            return fs  # The parent-dir is a root so they must have no related document.
        if self._commit_tracker.is_directory_added(fs.accessor_path):
            return fs  # The parent has recently been added.
        # Check for the parent-documents upwards until we determine where the sync is valid from.
        parent_paths = [(fs.accessor_path, fs.posix_path, fs.rel)]
        # NOTE: This can be changed to always create the directory doc for the root, if that is desired.
        i = len(paths) + p_idx
        assert i > 0
        while True:
            docs = await self._get_exact_docs(parent_paths[-1][0])
            if not docs:
                pass  # document missing, check its parent
            elif len(docs) > 1 or docs[0]["file"] is not False:
                # Existing parent document is invalid or not a directory, remove it to make it missing.
                await self._drop_document_tree(parent_paths[-1][0])
                # Since it's parent is implied to be an existing directory document or nothing in case of root, the
                # next cycle will exit and create the missing parent(s) including the one we've just deleted.
                # Solr documents will be queried for sync afterwards, so the removed documents must be committed.
                await self._solr_soft_commit()
            else:
                # parent exists
                if len(parent_paths) == 1:
                    return fs  # all parents exist because deepest exists
                break
            # Step up one level.
            if i == 1:  # root has no document, so it needs no checking
                parent_paths.append((create_accessor_path(root_path, paths[0][1]), paths[0][0].as_posix(), paths[0][1]))
                break
            i -= 1
            parent_paths.append((create_accessor_path(root_path, paths[i][1]), paths[i][0].as_posix(), paths[i][1]))
        # Create missing parent documents of directories, so the sync can be done.
        for i in range(len(parent_paths) - 2, -1, -1):
            await self._add_directory(parent_paths[i + 1][0], parent_paths[i][0], parent_paths[i][1])
        return fs

    async def _sync_execute(self, fs: SyncFilesystemParent, fs_listed: bool, docs: List[Tuple[str, dict]]):
        """
        Execute synchronization between prepared fs items and docs.
        At the beginning every solr doc is ensured to be up to date. (_sync_dir and _sync_file calls check-commit as
        necessary before invoking execute) Unfortunately there are a few cases where a document may change during the
        synchronization. These involve the document to have multiple alternative names under the current directory or
        one of its sub-directories when deletion is involved:
            - If there are multiple names for a document in the current directory (by symlinks) the first name could
              update the document which the latter will not see automatically. This could cause redundant updates of
              the same document.
            - When an alternative name is deleted, but the kept name is updated the order of operations won't have
              an affect on the result, but the latter sync will work based on an out of date document. This can also
              happen if one of the multiple names a document has is within a deleted sub-directory of the current
              directory.
            - If the multiple names of a document (maybe some in sub-directories) are all deleted the last delete
              handle will receive an operation error because the last accessor_path is not removable and it will
              need to explicitly remove the document.
            - A variant of losing all of the names for a document is when all indexed accessor paths are symlinks,
              they are all deleted and a new one is created. To avoid a consistency violation, removals may never
              delete a document proactively, only in face of an operation error.
        On the bright side, none of these result in consistency issues ultimately. It seems possible to optimize the
        redundant updates by commit-checking the documents and refreshing them.
        """
        fs_items = fs.items  # bind a local reference, so attr lookups are avoided
        # sort the files and docs so we can merge them iteratively
        fs_items.sort(key=lambda x: x.name)
        docs.sort(key=lambda x: x[0])
        # start syncing the two sources
        i = j = 0
        while i < len(fs_items) and j < len(docs):
            fs_item = fs_items[i]
            if self.IGNORE_TEMPORARY_NAME_RE.match(fs_item.name) is not None and stat.S_ISREG(fs_item.stat.st_mode):
                i += 1
                continue
            doc_name, doc = docs[j]
            if fs_item.name == doc_name:
                # there is a document in the index that corresponds to the fs item
                if stat.S_ISDIR(fs_item.stat.st_mode):
                    if doc["file"]:
                        # The fs item used to be a file but it is a directory now. It's got to be removed from the
                        # index. Its children documents will be recursively synced by the directory traversal.
                        # NOTE: removing the exact document would be enough here, but this is also correct
                        await self._drop_document_tree(fs_item.accessor_path, fs=fs)  # File-doc drop updates parent.
                        await self._add_directory(fs.accessor_path, fs_item.accessor_path, fs_item.posix_path)
                elif not doc["file"]:
                    # Replace directory document (and subtree) with the file document.
                    await self._drop_document_tree(fs_item.accessor_path)  # Dir-doc drop does not update parent.
                    await self._add_file(fs, fs_item)
                elif doc.get("size") != fs_item.stat.st_size or doc.get("mtime") != fs_item.stat.st_mtime_ns:
                    # The doc in the index is out of date. It has to be re-indexed.
                    await self._add_file(fs, fs_item)
                # Else the indexed document is considered up to date. However, take note that the there is no facility
                # to efficiently ensure that the content of the file is indeed what is indexed. Specifically, if the
                # file is replaced with another one with same length, but different content, while their modification
                # timestamp is the same, then the new content will not be indexed. This is a tradeoff we're willing to
                # make on the basis, that we do not really have a viable alternate implementation to consider.
                i += 1
                j += 1
            elif fs_item.name < doc_name:
                # no corresponding document for fs item
                if stat.S_ISDIR(fs_item.stat.st_mode):
                    # add the directory item
                    await self._add_directory(fs.accessor_path, fs_item.accessor_path, fs_item.posix_path)
                else:
                    # add the file to the index
                    await self._add_file(fs, fs_item)
                i += 1
            else:
                # no corresponding fs item for document
                # The directory listing is implied to be successful here, so use immediate subtree removal.
                await self._drop_document_tree(
                    create_accessor_path(fs.root_path, fs.rel, doc_name), fs=fs if doc["file"] else None
                )  # File-doc drop updates parent while dir-doc does not.
                j += 1
            self._handle_processing_interrupt(self.SYNC_INTERRUPT_CONTEXT_BOTH)
        while i < len(fs_items):
            # no corresponding document for fs item
            fs_item = fs_items[i]
            if stat.S_ISDIR(fs_item.stat.st_mode):
                # add the directory item
                await self._add_directory(fs.accessor_path, fs_item.accessor_path, fs_item.posix_path)
            else:
                # add the file to the index
                await self._add_file(fs, fs_item)
            i += 1
            self._handle_processing_interrupt(self.SYNC_INTERRUPT_CONTEXT_INSERT)
        # The sync can get here when there are no filesystem items in the directory. This *could* mean that the mount
        # or symlinked network drive or whatever loosely joined filesystem accessed on this path is temporarily
        # inaccessible. Since we can't be sure, we'll mark these documents as lost and only remove them after some
        # consecutive attempts give the same result. If there is any item in the directory we assume it is fully
        # accessible, so removal is instantaneous in that case.
        # Another factor to keep in mind is when a directory cannot be listed because of a permission error. Then even
        # if the directory is empty immediate removal shall be done. (see _sync_path())
        if docs and not fs_items and fs_listed:
            # Directory listing was successful, but empty. Use delayed removal on the parent path for conciseness.
            if not await self._lose_document_tree(fs.root_path, fs.rel):
                return  # delay removal of items until lose conditions are fulfilled
        while j < len(docs):
            # no corresponding fs item for document
            doc_name, doc = docs[j]
            await self._drop_document_tree(
                create_accessor_path(fs.root_path, fs.rel, doc_name), fs=fs if doc["file"] else None
            )  # File-doc drop updates parent while dir-doc does not.
            j += 1
            self._handle_processing_interrupt(self.SYNC_INTERRUPT_CONTEXT_REMOVE)

    async def _add_directory(self, parent_accessor_path: str, accessor_path: str, posix_path: str):
        """
        Directories must also be added to the index so cleanup of old documents is possible by parent-children
        traversal.
        Parent documents may get updates while its direct children are being indexed. For now this only extends
        to the "mtime" field. A directory does not have it natively, so it is re-purposed to provide reliable cache
        invalidation for services that aggregate data on a directory level.
        """
        # Create indexed document.
        d = {
            **self._template_document,  # fill unused fields with empty values
            "path": posix_path,
            "file": False,
            "accessors": {"add-distinct": accessor_path},
            "size": None,
            "mtime": None,
            "img_data": None,
            "img_bucket": None,
        }
        # Submit document to Solr.
        await self._transact_document(parent_accessor_path, [d])
        self._commit_tracker.add_directory(accessor_path)

    async def _add_file(self, fs: SyncFilesystemParent, fs_item: SyncFilesystemItem):
        """
        The fs item for the resolved accessor has no associated document or it is out of date.
        The path is resolved by chroot and accessor is made from the root and the rel-path.
        It could have many qualities that will inhibit processing:
            - can become non existent (my move, remove or broken symlink)
            - it can become a directory even, by remove & replace
            - we may not have or could lose permission for reading it
        """
        if fs_item.posix_path != convert_accessor_to_path(fs_item.accessor_path) and await self._get_docs(
            create_doc_query_path(fs_item.posix_path), fields="accessors"
        ):
            # If this file sync is not by the primary path and the document already exists, just add the alias.
            # NOTE: We do not set commit-within timeout for aliases here for optimization.
            await self._transact_document(
                fs.accessor_path, [{"path": fs_item.posix_path, "accessors": {"add-distinct": fs_item.accessor_path}}]
            )
            return
        self._notify_work(fs_item.path.name)
        # Check content extraction limits.
        summary = fs_item.path.name
        text = img_data = img_bucket = None
        if fs_item.stat.st_size > self.ec.indexer_extract_size_limit:
            logger.info("Skipping extraction because of size: %s", fs_item.path)
        else:
            try:
                with coerce_platform_os_error():
                    try:
                        async with fs_item.create_extraction_source(self.session, self.ec).prepare() as fes:
                            if fes.extract_text:  # Text-content extraction.
                                ex: TikaExtraction = await self._extract_content(fes, self._tika_extractor)
                                summary = ex.get_summary()
                                text = ex.get_content()
                            if fes.extract_image:  # Create thumbnail.
                                ex: PicotureExtraction = await self._extract_content(fes, self._picoture_extractor)
                                img_data = ex.get_image()
                                if img_data:
                                    img_data = pybase64.b64encode(img_data).decode("ascii")
                                    # NOTE: Bucketing is not fully implemented, right now it is effectively a boolean
                                    # to show there is image data.
                                    img_bucket = 1
                    except NtOSError as e:
                        # NOTE: We treat some OS errors as final even though their reason may be varied. For example:
                        #   - Google-docs links in a mounted Google Drive raise [Errno 22]/EINVAL for reading.
                        if e.errno != errno.EINVAL:
                            raise e
            except (FileNotFoundError, IsADirectoryError, NotADirectoryError):
                # The file may have been removed/moved/replaced since the iterdir call.
                raise SyncInterrupted()  # interrupt and retry with new sync-execute
            except PermissionError:  # no permission to read the file
                pass
            except OSError:  # handling unexpected OSErrors will prevent the indexing to get stuck on a file at least
                logger.exception("Unexpected error while extracting content from file")
        # Detect the language to put the content into.
        field_lang = self._lang_det.get_primary_language(text)
        if field_lang not in self.languages:
            field_lang = "faba"
        # Create indexed document.
        d = {
            **self._template_document,  # fill unused fields with empty values
            "path": fs_item.posix_path,
            "file": True,
            "accessors": {"add-distinct": fs_item.accessor_path},
            "size": fs_item.stat.st_size,
            "mtime": fs_item.stat.st_mtime_ns,
            f"summary_{field_lang}": summary,
            f"text_{field_lang}": text,
            "img_data": img_data,
            "img_bucket": img_bucket,
        }
        # Submit document and parent update to Solr.
        await self._transact_document(
            fs.accessor_path,
            serialize_solr_update_json([("add", d), ("add", fs.create_parent_directory_update())]),
            commit_within=2000,
        )
        self._commit_tracker.add_commit_time(2.0)

    async def _extract_content(self, source: AbstractExtractionSource, extractor: AbstractContentExtractor):
        attempt = 0  # Handle extractor failures and retries.
        while True:
            try:
                return await extractor.extract(source)
            except ServerInternalExtractError as e:  # no retry
                logger.info("Abandoning extraction of: %s", source.describe(), exc_info=True)
                return e.extraction
            except RetryExtractError as e:
                attempt += 1
                if attempt <= self.IMPLICIT_EXTRACT_RETRY_ATTEMPTS:
                    await asyncio.sleep(0.5)  # implicit retry delay so the extractor may recover
                    continue  # retry extraction
                logger.info("Abandoning extraction of: %s", source.describe(), exc_info=True)
                return e.extraction

    async def _transact_document(
        self,
        parent_accessor_path: str,
        payload: Union[List[dict], dict, bytes],
        commit_within: Optional[int] = None,
    ):
        """
        Track documents with transaction.
        """
        try:
            async with nano_update_post(
                self.session, self.ec.solr_update_url, self.ec.solr_auth_header, payload, commit_within=commit_within
            ) as r:
                result, success = await preparse_solr_response(r)
            if not success:
                raise SolrRequestFailed(f"Add doc failed\n{result}")
        finally:
            self._commit_tracker.track(parent_accessor_path)

    async def _lose_document_tree(self, root: str, path: str) -> bool:
        """
        The fs item for the resolved accessor was found missing. Add/update lost record and return true if the
        lose condition is fulfilled and the caller may drop the documents.
        """
        t, c = await self.index_state_db.update_lost_record(root, path)
        return c >= self.ec.indexer_remove_lost_count and (time.time() - t) > self.ec.lost_remove_minimum_delay

    async def _drop_document_tree(
        self, accessor_path: str, keep_exact: bool = False, fs: Optional[SyncFilesystemParent] = None
    ):
        """
        Delete all documents under the path then the path-doc itself if desired.
        """
        await self._drop_document_subtree(accessor_path, [0])
        if not keep_exact:
            docs = await self._get_exact_docs(accessor_path)
            for doc in docs:
                await self._drop_document_accessors(doc["path"], [accessor_path], fs)

    async def _drop_document_subtree(self, parent_accessor_path: str, docs_processed: List[int]):
        """
        Document removal is not trivial because it must keep working consistency in face of intermediate or partial
        errors and interruption and reliable resumption. These require serializable consistency on the documents that
        will be collected in a depth-first fashion.
        """
        try:
            async for docs in query_iter_docs(
                self.session,
                self.ec.solr_query_url,
                self.ec.solr_auth_header,
                create_docs_query_children(parent_accessor_path),
                "path asc",
                "path,file,accessors",
                self.DOC_SLICE_DROP_TREE,
                exc_cls=SolrRequestFailed,
            ):
                for doc in docs:
                    accessors_to_drop = []
                    for acc in doc["accessors"]:
                        acc_parent, _, acc_name = acc.rpartition("/")
                        if acc_parent == parent_accessor_path and acc_name:
                            accessors_to_drop.append(acc)
                    if not accessors_to_drop:
                        # TODO: INCON-ACCS
                        raise Exception("Inconsistent accessors for drop")
                    if not doc["file"]:
                        for accessor_to_drop in accessors_to_drop:
                            await self._drop_document_subtree(accessor_to_drop, docs_processed)
                    await self._drop_document_accessors(doc["path"], accessors_to_drop)
                    docs_processed[0] += 1
                    if docs_processed[0] % self.DOC_SLICE_DROP_TREE == 0:
                        self._handle_processing_interrupt(self.SYNC_INTERRUPT_CONTEXT_DELETE)
        finally:
            self._commit_tracker.track(parent_accessor_path, affects_downstream=True)

    async def _drop_document_accessors(
        self, path: str, accessors: List[str], fs: Optional[SyncFilesystemParent] = None
    ):
        """
        Remove the accessors from the document identified by path. When it would lose its last accessor the whole
        document is deleted instead.
        """
        async with nano_update_post(
            self.session,
            self.ec.solr_update_url,
            self.ec.solr_auth_header,
            [{"path": path, "accessors": {"remove": acc}} for acc in accessors],
        ) as r:
            result, success = await preparse_solr_response(r)
        if success:
            if any(path == convert_accessor_to_path(acc) for acc in accessors):
                # When the primary accessor is deleted, all other aliases shall be cleaned up for complete consistency.
                # That would happen eventually, at the next poll-traversal, but we can expedite it here. The remaining
                # accessors must be broken symlinks, so we generate internal sync-events for them.
                for doc in await self._get_docs(create_doc_query_path(path), fields="accessors"):
                    for acc in doc["accessors"]:
                        try:
                            # NOTE: No event delay is necessary on Windows here, so not using the exists-meta.
                            self._event_queue.put_nowait(SchedulerFilesystemEvent(convert_accessor_to_path(acc)))
                        except asyncio.QueueFull:
                            logger.warning("Expedited symlink cleanup event dropped because queue is full")
            return
        error = get_solr_error(result)
        if not error or error[0] != 400 or not error[1]:
            raise SolrRequestFailed(f"Delete accessor from doc failed (0)\n{result}")
        if error[1].endswith(" missing required field: accessors"):
            pass  # cannot remove last accessor, go to document removal
        elif error[1].endswith(" missing required field: file"):
            assert False, "this should not happen if commit-checking is correct and can be enforced"
            return  # document is already removed
        else:
            raise SolrRequestFailed(f"Delete accessor from doc failed (1)\n{error}\n{result}")
        # remove the whole document
        commands = {"delete": {"id": path}}
        if fs:
            commands["add"] = {"doc": fs.create_parent_directory_update()}
        async with nano_update_post(self.session, self.ec.solr_update_url, self.ec.solr_auth_header, commands) as r:
            result, success = await preparse_solr_response(r)
        if not success:
            raise SolrRequestFailed(f"Delete doc failed\n{result}")

    async def _solr_check_commit(self, accessor_path):
        if self._commit_tracker.check(accessor_path):
            await self._solr_soft_commit()

    async def _solr_soft_commit(self):
        """
        Request a soft-commit from solr, because we need to have up-to-date information for in-flight docs.
        """
        async with nano_update_post(
            self.session, self.ec.solr_update_url, self.ec.solr_auth_header, {}, soft_commit=True
        ) as r:
            result, success = await preparse_solr_response(r)
        if not success:
            raise SolrRequestFailed(f"Soft commit failed\n{result}")
        self._commit_tracker.clear()

    async def _get_exact_docs(self, accessor_path: str) -> List[dict]:
        """
        Convenience function.
        Some matched documents are not what we're looking for, but the index tokenization cannot filter more
        precisely. The extra document could already be deleted. For more information see the docstring of
        create_docs_query_exact().
        Additionally the document for the root needs a special branch.
        """
        docs = await self._get_docs(
            create_docs_query_exact(*convert_accessor_to_parent_accessor(accessor_path)), "path,file,accessors"
        )
        return [doc for doc in docs if accessor_path in doc["accessors"]]

    async def _get_docs(
        self, query_expr: str, fields: str = "path,file,accessors,size,mtime", rows: int = DOC_SLICE_ITER_DIR
    ) -> List[dict]:
        """
        Get ALL solr documents by a query with cursor pagination.
        """
        docs = []
        async for ds in query_iter_docs(
            self.session,
            self.ec.solr_query_url,
            self.ec.solr_auth_header,
            query_expr,
            "path asc",
            fields,
            rows,
            exc_cls=SolrRequestFailed,
        ):
            docs.extend(ds)
        return docs
