logfile_sub_source.hh

/**
 * Copyright (c) 2007-2012, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * @file logfile_sub_source.hh
 */

#ifndef logfile_sub_source_hh
#define logfile_sub_source_hh

#include <array>
#include <atomic>
#include <deque>
#include <exception>
#include <functional>
#include <map>
#include <unordered_map>
#include <utility>
#include <vector>

#include <limits.h>

#include "base/time_util.hh"
#include "big_array.hh"
#include "bookmarks.hh"
#include "document.sections.hh"
#include "filter_observer.hh"
#include "log_format.hh"
#include "logfile.hh"
#include "strong_int.hh"
#include "textview_curses.hh"

STRONG_INT_TYPE(uint64_t, content_line);

struct sqlite3_stmt;
extern "C"
{
int sqlite3_finalize(sqlite3_stmt* pStmt);
}

class logfile_sub_source;

struct breakpoint_info {
    enum class source_type : int {
        src_location,
        message_schema,
    };

    std::string bp_description;
    source_type bp_source{source_type::src_location};
    bool bp_enabled{true};
};

class index_delegate {
public:
    virtual ~index_delegate() = default;

    virtual void index_start(logfile_sub_source& lss) {}

    virtual void index_line(logfile_sub_source& lss,
                            logfile* lf,
                            logfile::iterator ll)
    {
    }

    virtual void index_complete(logfile_sub_source& lss) {}
};

class sql_filter : public text_filter {
public:
    sql_filter(logfile_sub_source& lss,
               std::string stmt_str,
               sqlite3_stmt* stmt)
        : text_filter(EXCLUDE, filter_lang_t::SQL, std::move(stmt_str), 0),
          sf_log_source(lss)
    {
        this->sf_filter_stmt = stmt;
    }

    /**
     * Handed out in creation order and never reused, so a filter allocated
     * where a freed one used to live is still told apart from it.
     */
    static uint64_t next_serial()
    {
        static std::atomic<uint64_t> counter{0};

        return counter.fetch_add(1, std::memory_order_relaxed) + 1;
    }

    bool matches(std::optional<line_source> ls,
                 const shared_buffer_ref& line) override;

    std::string to_command() const override;

    /**
     * @return This thread's copy of the statement, prepared on first use, or
     * null if it could not be prepared.
     */
    sqlite3_stmt* stmt_for_this_thread();

    auto_mem<sqlite3_stmt> sf_filter_stmt{sqlite3_finalize};
    logfile_sub_source& sf_log_source;
    const uint64_t sf_serial{next_serial()};
};

class log_location_history : public location_history {
public:
    explicit log_location_history(logfile_sub_source& lss)
        : llh_history(std::begin(this->llh_backing),
                      std::end(this->llh_backing)),
          llh_log_source(lss)
    {
    }

    ~log_location_history() override = default;

    void loc_history_append(vis_line_t top) override;

    std::optional<vis_line_t> loc_history_back(vis_line_t current_top) override;

    std::optional<vis_line_t> loc_history_forward(
        vis_line_t current_top) override;

private:
    nonstd::ring_span<content_line_t> llh_history;
    logfile_sub_source& llh_log_source;
    content_line_t llh_backing[MAX_SIZE];
};

class logline_window;

/**
 * Delegate class that merges the contents of multiple log files into a single
 * source of data for a text view.
 */
class logfile_sub_source
    : public text_sub_source
    , public text_time_translator
    , public text_accel_source
    , public text_anchors
    , public text_mark_scanner
    , public text_delegate
    , public text_detail_provider
    , public lnav_config_listener {
public:
    const static bookmark_type_t BM_FILES;

    virtual void text_filters_changed();

    logfile_sub_source();

    ~logfile_sub_source() = default;

    enum class line_context_t : uint8_t {
        filename,
        basename,
        none,
        time_column,
    };

    void increase_line_context();

    bool decrease_line_context();

    size_t get_filename_offset() const;

    line_context_t get_line_context() const { return this->lss_line_context; }

    void set_force_rebuild() { this->lss_force_rebuild = true; }

    bool is_rebuild_forced() const { return this->lss_force_rebuild; }

    /**
     * While set, rebuild_index() indexes the files but does not merge their
     * lines into the combined index.
     */
    void set_merge_deferred(bool val) { this->lss_merge_deferred = val; }

    bool is_merge_deferred() const { return this->lss_merge_deferred; }

    bool list_input_handle_key(listview_curses& lv, const ncinput& ch);

    void set_marked_only(bool val)
    {
        if (this->lss_marked_only != val) {
            this->lss_marked_only = val;
            this->text_filters_changed();
        }
    }

    void update_filter_hash_state(hasher& h) const;

    bool get_marked_only() { return this->lss_marked_only; }

    size_t text_line_count() { return this->lss_filtered_index.size(); }

    size_t text_line_width(textview_curses& curses)
    {
        return this->lss_longest_line;
    }

    size_t file_count() const;

    bool empty() const { return this->lss_filtered_index.empty(); }

    line_info text_value_for_line(textview_curses& tc,
                                  int row,
                                  std::string& value_out,
                                  line_flags_t flags);

    void text_attrs_for_line(textview_curses& tc,
                             int row,
                             string_attrs_t& value_out);

    void text_horiz_columns(textview_curses& tc,
                            vis_line_t start_row,
                            vis_line_t end_row,
                            std::set<int>& columns_out);

    size_t text_size_for_line(textview_curses& tc, int row, line_flags_t flags);

    void text_mark(const bookmark_type_t* bm, vis_line_t line, bool added);

    void text_clear_marks(const bookmark_type_t* bm);

    bool insert_file(const std::shared_ptr<logfile>& lf);

    void remove_file(std::shared_ptr<logfile> lf);

    enum class rebuild_result {
        rr_in_progress,
        rr_no_change,
        rr_appended_lines,
        rr_partial_rebuild,
        rr_full_rebuild,
    };

    rebuild_result rebuild_index(std::optional<ui_clock::time_point> deadline
                                 = std::nullopt);

    void text_update_marks(vis_bookmarks& bm);

    void set_user_mark(const bookmark_type_t* bm, content_line_t cl)
    {
        this->lss_user_marks[bm].insert_once(cl);
    }

    bookmarks<content_line_t>::type& get_user_bookmarks()
    {
        return this->lss_user_marks;
    }

    std::map<std::string, breakpoint_info>& get_breakpoints()
    {
        return this->lss_breakpoints;
    }

    const std::map<std::string, breakpoint_info>& get_breakpoints() const
    {
        return this->lss_breakpoints;
    }

    bookmark_metadata& get_bookmark_metadata(content_line_t cl);

    bookmark_metadata& get_bookmark_metadata(vis_line_t vl)
    {
        return this->get_bookmark_metadata(this->at(vl));
    }

    struct bookmark_metadata_context {
        std::optional<vis_line_t> bmc_current;
        std::optional<bookmark_metadata*> bmc_current_metadata;
        std::optional<vis_line_t> bmc_next_line;
    };

    bookmark_metadata_context get_bookmark_metadata_context(
        vis_line_t vl,
        bookmark_metadata::categories desired
        = bookmark_metadata::categories::any) const;

    std::optional<bookmark_metadata*> find_bookmark_metadata(
        content_line_t cl) const;

    std::optional<bookmark_metadata*> find_bookmark_metadata(
        vis_line_t vl) const
    {
        if (vl >= vis_line_t(this->lss_filtered_index.size())) {
            return std::nullopt;
        }
        return this->find_bookmark_metadata(this->at(vl));
    }

    void erase_bookmark_metadata(content_line_t cl);

    void erase_bookmark_metadata(vis_line_t vl)
    {
        this->erase_bookmark_metadata(this->at(vl));
    }

    void clear_bookmark_metadata();

    int get_filtered_count() const
    {
        return this->lss_index.size() - this->lss_filtered_index.size();
    }

    // Full index size (includes sibling metric rows that are suppressed
    // from lss_filtered_index).
    size_t index_size() const { return this->lss_index.size(); }

    content_line_t content_line_at(size_t idx) const
    {
        return this->lss_index[idx].value();
    }

    // True when the row at the given `lss_index` position is excluded
    // by the active filters.  Lets callers walking `lss_index` directly
    // (e.g. sibling fan-out) distinguish "in lss_index but suppressed
    // by dedup" from "in lss_index but hidden by filter".  Callers
    // that check many rows should cache the filter masks via
    // `get_enabled_filter_mask()` and pass them in rather than letting
    // this function re-walk the filter stack per call.
    bool row_is_filtered_out(size_t idx,
                             uint32_t filter_in_mask,
                             uint32_t filter_out_mask);

    // Snapshot the enabled filter masks.  Cheap but not free — walks
    // the filter stack.  Safe to cache for the duration of a tight
    // loop over rows.
    void get_enabled_filter_mask(uint32_t& filter_in_mask,
                                 uint32_t& filter_out_mask)
    {
        this->get_filters().get_enabled_mask(filter_in_mask, filter_out_mask);
    }

    int get_filtered_count_for(size_t filter_index) const;

    Result<void, lnav::console::user_message> set_sql_filter(
        std::string stmt_str, sqlite3_stmt* stmt);

    Result<void, lnav::console::user_message> set_sql_marker(
        std::string stmt_str, sqlite3_stmt* stmt);

    /**
     * Whether the expression has to be one that depends only on the message.
     * Required of a filter, whose result is evaluated once at index time and
     * then remembered; not of a marker, which is re-evaluated over every
     * message each time it is set.
     */
    enum class expr_purity {
        required,
        not_required,
    };

    Result<void, lnav::console::user_message> set_preview_sql_filter(
        sqlite3_stmt* stmt, expr_purity purity = expr_purity::required);

    std::string get_sql_filter_text()
    {
        auto filt = this->get_sql_filter();

        if (filt) {
            return filt.value()->get_id();
        }
        return "";
    }

    std::optional<std::shared_ptr<text_filter>> get_sql_filter();

    std::string get_sql_marker_text() const
    {
        return this->lss_marker_stmt_text;
    }

    std::shared_ptr<logfile> find(const char* fn, content_line_t& line_base);

    std::shared_ptr<logfile> find(content_line_t& line) const
    {
        std::shared_ptr<logfile> retval;

        retval = this->lss_files[line / MAX_LINES_PER_FILE]->get_file();
        line = content_line_t(line % MAX_LINES_PER_FILE);

        return retval;
    }

    logfile* find_file_ptr(content_line_t& line) const
    {
        auto* retval
            = this->lss_files[line / MAX_LINES_PER_FILE]->get_file_ptr();
        line = content_line_t(line % MAX_LINES_PER_FILE);

        return retval;
    }

    logline* find_line(content_line_t line) const
    {
        logline* retval = nullptr;
        auto lf = this->find_file_ptr(line);

        if (lf != nullptr) {
            auto ll_iter = lf->begin() + line;

            retval = &(*ll_iter);
        }

        return retval;
    }

    std::optional<std::pair<std::shared_ptr<logfile>, logfile::iterator>>
    find_line_with_file(content_line_t line) const
    {
        std::shared_ptr<logfile> lf = this->find(line);

        if (lf != nullptr) {
            auto ll_iter = lf->begin() + line;

            return std::make_pair(lf, ll_iter);
        }

        return std::nullopt;
    }

    std::optional<std::pair<std::shared_ptr<logfile>, logfile::iterator>>
    find_line_with_file(std::optional<vis_line_t> vl) const
    {
        if (vl && vl.value() >= 0_vl
            && vl.value() < vis_line_t(this->lss_filtered_index.size()))
        {
            return this->find_line_with_file(this->at(vl.value()));
        }

        return std::nullopt;
    }

    std::optional<vis_line_t> find_from_time(const timeval& start) const;

    std::optional<vis_line_t> find_from_time(time_t start) const
    {
        const auto tv = timeval{start, 0};

        return this->find_from_time(tv);
    }

    std::optional<vis_line_t> find_from_time(const exttm& etm) const
    {
        return this->find_from_time(etm.to_timeval());
    }

    std::optional<vis_line_t> find_from_content(content_line_t cl);

    std::optional<row_info> time_for_row(vis_line_t row)
    {
        if (row >= 0_vl && row < (ssize_t) this->lss_filtered_index.size()) {
            auto cl = this->at(row);
            return row_info{
                this->find_line(cl)->get_timeval(),
                (int64_t) cl,
            };
        }
        return std::nullopt;
    }

    std::optional<vis_line_t> row_for(const row_info& ri);

    std::optional<vis_line_t> row_for_time(struct timeval time_bucket)
    {
        return this->find_from_time(time_bucket);
    }

    content_line_t at(vis_line_t vl) const
    {
        return this->lss_index[this->lss_filtered_index[vl]].value();
    }

    // Position of a visible row within the raw `lss_index` — lets
    // callers walk forward past the row to reach metric siblings
    // that were suppressed from the filtered index.
    size_t index_at(vis_line_t vl) const
    {
        return this->lss_filtered_index[vl];
    }

    size_t get_filtered_before() const
    {
        return this->lss_filtered_index.empty() ? 0
                                                : this->lss_filtered_index[0];
    }

    size_t get_filtered_after() const
    {
        if (this->lss_filtered_index.empty()) {
            return 0;
        }

        return this->lss_index.size() - this->lss_filtered_index.back() - 1;
    }

    content_line_t at_base(vis_line_t vl)
    {
        while (vl > 0_vl
               && this->find_line(this->at(vl))->get_sub_offset() != 0)
        {
            --vl;
        }

        return this->at(vl);
    }

    std::unique_ptr<logline_window> window_at(vis_line_t start_vl,
                                              vis_line_t end_vl);

    std::unique_ptr<logline_window> window_at(vis_line_t start_vl);

    std::unique_ptr<logline_window> window_to_end(vis_line_t start_vl);

    /**
     * Container for logfile references that keeps of how many lines in the
     * logfile have been indexed.
     */
    struct logfile_data {
        logfile_data(size_t index,
                     filter_stack& fs,
                     const std::shared_ptr<logfile>& lf)
            : ld_file_index(index), ld_filter_state(fs, lf),
              ld_visible(lf->is_indexing())
        {
            lf->set_logline_observer(&this->ld_filter_state);
            this->ld_file_ptr = lf.get();
        }

        void clear()
        {
            this->ld_filter_state.lfo_filter_state.clear();
            this->ld_file_ptr = nullptr;
        }

        void set_file(const std::shared_ptr<logfile>& lf)
        {
            this->ld_filter_state.lfo_filter_state.tfs_logfile = lf;
            this->ld_file_ptr = lf.get();
            this->ld_lines_indexed = 0;
            this->ld_lines_watched = 0;
            this->ld_visible = lf->is_indexing();
            lf->set_logline_observer(&this->ld_filter_state);
        }

        std::shared_ptr<logfile> get_file() const
        {
            return this->ld_filter_state.lfo_filter_state.tfs_logfile;
        }

        logfile* get_file_ptr() const { return this->ld_file_ptr; }

        bool is_visible() const
        {
            return this->get_file_ptr() != nullptr && this->ld_visible;
        }

        void set_visibility(bool vis) { this->ld_visible = vis; }

        size_t ld_file_index;
        line_filter_observer ld_filter_state;
        size_t ld_lines_indexed{0};
        size_t ld_lines_watched{0};
        logfile* ld_file_ptr{nullptr};
        bool ld_visible;
    };

    using iterator = std::vector<std::unique_ptr<logfile_data>>::iterator;
    using const_iterator
        = std::vector<std::unique_ptr<logfile_data>>::const_iterator;

    size_t size() const { return this->lss_files.size(); }

    iterator begin() { return this->lss_files.begin(); }

    iterator end() { return this->lss_files.end(); }

    const_iterator cbegin() const { return this->lss_files.begin(); }

    const_iterator cend() const { return this->lss_files.end(); }

    iterator find_data(content_line_t& line)
    {
        auto retval = this->lss_files.begin();
        std::advance(retval, line / MAX_LINES_PER_FILE);
        line = content_line_t(line % MAX_LINES_PER_FILE);

        return retval;
    }

    iterator find_data(content_line_t line, uint64_t& offset_out)
    {
        auto retval = this->lss_files.begin();
        std::advance(retval, line / MAX_LINES_PER_FILE);
        offset_out = line % MAX_LINES_PER_FILE;

        return retval;
    }

    std::optional<logfile_data*> find_data(const std::shared_ptr<logfile>& lf)
    {
        for (auto& ld : *this) {
            if (ld->ld_filter_state.lfo_filter_state.tfs_logfile == lf) {
                return ld.get();
            }
        }
        return std::nullopt;
    }

    iterator find_data_i(const std::shared_ptr<const logfile>& lf)
    {
        for (auto iter = this->begin(); iter != this->end(); ++iter) {
            if ((*iter)->ld_filter_state.lfo_filter_state.tfs_logfile == lf) {
                return iter;
            }
        }

        return this->end();
    }

    content_line_t get_file_base_content_line(iterator iter)
    {
        ssize_t index = std::distance(this->begin(), iter);

        return content_line_t(index * MAX_LINES_PER_FILE);
    }

    // The file a content line came from, as an index into lss_files.  The
    // inverse of get_file_base_content_line().
    static size_t file_index_for(content_line_t cl)
    {
        return cl / MAX_LINES_PER_FILE;
    }

    /**
     * Called on the calling thread while a parallel scan is in flight so the
     * UI keeps moving.  `off` and `total` are summed over the files being
     * scanned.
     *
     * The workers are writing those files while this runs, so an
     * implementation must not read them -- the numbers handed in here are all
     * it gets.  `in_flight` carries a per-file breakdown of the same reading
     * for the files that have not finished, so the file list can be drawn
     * without touching them either.  Returning interrupt stops the scan as
     * soon as the workers notice.
     */
    using scan_progress_fn = std::function<lnav::progress_result_t(
        file_off_t, file_ssize_t, const std::vector<index_progress_report>&)>;

    void set_scan_progress(scan_progress_fn fn)
    {
        this->lss_scan_progress = std::move(fn);
    }

    void set_index_delegate(index_delegate* id)
    {
        if (id != this->lss_index_delegate) {
            this->lss_index_delegate = id;
            this->reload_index_delegate();
        }
    }

    index_delegate* get_index_delegate() const
    {
        return this->lss_index_delegate;
    }

    void reload_index_delegate();

    class meta_grepper
        : public grep_proc_source<vis_line_t>
        , public grep_proc_sink<vis_line_t> {
    public:
        meta_grepper(logfile_sub_source& source) : lmg_source(source) {}

        std::optional<line_info> grep_value_for_line(
            vis_line_t line, std::string& value_out) override;

        vis_line_t grep_initial_line(vis_line_t start) override;

        void grep_next_line(vis_line_t& line) override;

        void grep_quiesce() override;

        void grep_reset(grep_proc<vis_line_t>& gp,
                        vis_line_t start,
                        vis_line_t stop,
                        grep_pattern_mask_t patterns) override;

        void grep_begin(grep_proc<vis_line_t>& gp,
                        vis_line_t start,
                        vis_line_t stop,
                        grep_pattern_mask_t patterns) override;

        void grep_end(grep_proc<vis_line_t>& gp) override;

        void grep_match(grep_proc<vis_line_t>& gp,
                        vis_line_t line,
                        grep_pattern_mask_t patterns) override;

        logfile_sub_source& lmg_source;
        bool lmg_done{false};
    };

    std::optional<
        std::pair<grep_proc_source<vis_line_t>*, grep_proc_sink<vis_line_t>*>>
    get_grepper();

    std::optional<location_history*> get_location_history()
    {
        return &this->lss_location_history;
    }

    void text_crumbs_for_line(int line, std::vector<breadcrumb::crumb>& crumbs);

    bool text_handle_mouse(textview_curses& tc,
                           const listview_curses::display_line_content_t&,
                           mouse_event& me);

    Result<bool, lnav::console::user_message> eval_sql_filter(
        sqlite3_stmt* stmt, iterator ld, logfile::const_iterator ll);

    void invalidate_sql_filter();

    void set_line_meta_changed() { this->lss_line_meta_changed = true; }

    bool is_line_meta_changed() const { return this->lss_line_meta_changed; }

    void set_exec_context(exec_context* ec) { this->lss_exec_context = ec; }

    exec_context* get_exec_context() const { return this->lss_exec_context; }

    static constexpr unsigned CONTENT_LINE_BITS = 38;
    static constexpr uint64_t MAX_CONTENT_LINES = 1ULL << CONTENT_LINE_BITS;
    static constexpr uint64_t MAX_LINES_PER_FILE = logfile::MAX_LINES;
    static constexpr uint64_t MAX_FILES
        = (MAX_CONTENT_LINES / MAX_LINES_PER_FILE);

    std::function<void(logfile_sub_source&, file_off_t, file_size_t)>
        lss_sorting_observer;

    uint32_t lss_index_generation{0};

    /**
     * A change to the filtered index: every row from ic_from_row onward was
     * added or replaced, leaving ic_row_count rows.  Appends have ic_from_row
     * equal to the previous row count; rebuilds and filter changes start
     * earlier.  External clients get these through /api/poll so they can
     * re-query only the rows that changed.
     */
    struct index_change {
        uint64_t ic_seq;
        uint32_t ic_generation;
        size_t ic_from_row;
        size_t ic_row_count;
    };

    static constexpr size_t MAX_INDEX_CHANGES = 256;

    /** Sequence number of the latest entry in lss_index_changes. */
    uint64_t lss_index_change_seq{0};
    std::deque<index_change> lss_index_changes;

    void record_index_change(size_t from_row);

    void quiesce();

    struct __attribute__((__packed__)) indexed_content {
        enum class level_t : uint8_t {
            normal,
            warning,
            error,
        };

        static level_t level_from_log(const logfile::const_iterator iter)
        {
            if (!iter->is_message()) {
                return level_t::normal;
            }
            switch (iter->get_msg_level()) {
                case log_level_t::LEVEL_WARNING:
                    return level_t::warning;
                case log_level_t::LEVEL_ERROR:
                case log_level_t::LEVEL_FATAL:
                case log_level_t::LEVEL_CRITICAL:
                    return level_t::error;
                default:
                    return level_t::normal;
            }
        }

        indexed_content() = default;

        indexed_content(content_line_t cl, const logfile::const_iterator iter)
            : ic_value(cl),
              ic_level(lnav::enums::to_underlying(level_from_log(iter)))
        {
        }

        content_line_t value() const { return content_line_t(this->ic_value); }

        level_t level() const { return static_cast<level_t>(this->ic_level); }

        uint64_t ic_value : CONTENT_LINE_BITS;
        uint8_t ic_level : 2;
    };

    big_array<indexed_content> lss_index;

    /**
     * The marks that are a function of the index alone, and so can be counted
     * once per region instead of being held per row.  The user marks are left
     * out on purpose: they change without the index changing, which is the
     * one thing this table cannot absorb.
     */
    enum class region_mark_t : uint8_t {
        warning,
        error,
        file_start,

        RM__MAX
    };

    static constexpr size_t REGION_MARK_MAX
        = lnav::enums::to_underlying(region_mark_t::RM__MAX);

    /**
     * A running summary of a fixed-size window of rows.  Ruling a window out
     * costs one comparison per SIZE rows, which is what lets a scan stand in
     * for a set of marked rows that would otherwise have to be rebuilt in
     * full every time the view reloads.
     */
    struct index_region {
        // A power of two so the region for a row is a shift, and as large as
        // a uint16_t count can describe, since the only scan whose length
        // this bounds happens once per find_mark() call.
        static constexpr size_t SIZE = 8192;

        uint16_t ir_counts[REGION_MARK_MAX]{};

        uint16_t& count_for(region_mark_t rm)
        {
            return this->ir_counts[lnav::enums::to_underlying(rm)];
        }

        uint16_t count_for(region_mark_t rm) const
        {
            return this->ir_counts[lnav::enums::to_underlying(rm)];
        }
    };

    static_assert(index_region::SIZE <= UINT16_MAX,
                  "a region's counts have to fit in ir_counts");

    static size_t region_for_row(vis_line_t vl)
    {
        return static_cast<size_t>(vl) / index_region::SIZE;
    }

    size_t region_count() const { return this->lss_regions.size(); }

    const index_region& region_at(size_t region_index) const
    {
        return this->lss_regions[region_index];
    }

    /**
     * The number of marks of the given kind in the half-open row range
     * [start, stop).  Whole regions are taken from the summary table, so only
     * the rows at either end that fall inside a partially covered region are
     * actually looked at.
     */
    size_t count_marks(vis_line_t start,
                       vis_line_t stop,
                       region_mark_t rm) const;

    /**
     * The nearest row carrying the given mark, searching strictly after (or
     * before) `from` so that a caller can walk hit to hit the way
     * bookmark_vector::next() does.
     *
     * Regions whose count for the mark is zero are skipped without looking at
     * any of their rows.
     */
    std::optional<vis_line_t> find_mark(vis_line_t from,
                                        direction dir,
                                        region_mark_t rm) const;

    indexed_content::level_t level_for_row(vis_line_t vl) const
    {
        return this->lss_index[this->lss_filtered_index[vl]].level();
    }

    /**
     * Is this the first row that is displayed from its file?
     *
     * The file is the top bits of the row's content line, so this is two
     * loads and a shift against the preceding row -- no summary table and no
     * walk out to the logfile.  A row past the end of the view is not a
     * start, which is what keeps the last row of the last file rendering
     * without a bottom corner.
     */
    bool is_file_start(vis_line_t vl) const;

    bool row_has_mark(vis_line_t vl, region_mark_t rm) const;

    // Whether any row in the half-open range [start, stop) carries the mark.
    // The regions wholly inside the range are checked by count first, so a
    // wide range normally answers without looking at a single row -- which is
    // what the gutter needs, since it asks about a range per visible row on
    // every repaint.
    bool any_mark_in_range(vis_line_t start,
                           vis_line_t stop,
                           region_mark_t rm) const;

    // The mark types this source answers from `lss_regions` instead of from
    // the view's bookmark vectors.
    static std::optional<region_mark_t> region_mark_for(
        const bookmark_type_t* bt);

    bool text_scans_mark(const bookmark_type_t* bt) const
    {
        return region_mark_for(bt).has_value();
    }

    std::optional<vis_line_t> text_adjacent_mark(
        const bookmark_type_t* bt,
        vis_line_t from,
        text_anchors::direction dir) const
    {
        return this->find_mark(from, dir, region_mark_for(bt).value());
    }

    bool text_mark_at_row(const bookmark_type_t* bt,
                          vis_line_t vl) const
    {
        return this->row_has_mark(vl, region_mark_for(bt).value());
    }

    bool text_any_mark_in_range(const bookmark_type_t* bt,
                                vis_line_t start,
                                vis_line_t stop) const
    {
        return this->any_mark_in_range(
            start, stop, region_mark_for(bt).value());
    }

    std::optional<vis_line_t> row_for_anchor(const std::string& id);

    std::optional<vis_line_t> adjacent_anchor(vis_line_t vl, direction dir);

    std::optional<std::string> anchor_for_row(vis_line_t vl);

    std::unordered_set<std::string> get_anchors();

    std::optional<json_string> text_row_details(const textview_curses& tc);

    /**
     * Adds "index-seq" (lss_index_change_seq) so a script can read the change
     * log position in the same snapshot as its queries.
     */
    std::optional<std::string> text_view_details() const;

    void reload_config(error_reporter& reporter);

    bool is_indexing_in_progress() const
    {
        return this->lss_indexing_in_progress;
    }

    void clear_preview();

    void add_commands_for_session(
        const std::function<void(const std::string&)>& receiver);

    std::vector<highlighter> lss_highlighters;

protected:
    void text_accel_display_changed() { this->clear_line_size_cache(); }

    logline* text_accel_get_line(vis_line_t vl)
    {
        return this->find_line(this->at(vl));
    }

private:
    static const size_t LINE_SIZE_CACHE_SIZE = 512;

    /** What a parallel pre-scan worked out about one file. */
    struct prescan_result {
        logfile::rebuild_result_t psr_result{
            logfile::rebuild_result_t::NO_NEW_LINES};
        /**
         * The format's timestamp flags as they stood *before* the scan.  The
         * loop OR's these in ahead of the file's own scan, so caching them
         * here is what keeps lss_all_timestamp_flags exactly one pass behind,
         * as it has always been.
         */
        uint32_t psr_timestamp_flags{0};
        /**
         * Set when the scan threw.  The loop rethrows this in place of the
         * file's result, so the failure still comes out of the spot the scan
         * runs in on this thread.
         */
        std::exception_ptr psr_exception;
    };

    using prescan_map = std::unordered_map<const logfile*, prescan_result>;

    /**
     * Index this round's files up front and across several threads.
     *
     * rebuild_index() writes only its own logfile, so the files are
     * independent; the merge, the sort, and everything else the loop does
     * with the results stays on the calling thread and in file_order.
     * Returns an empty map when there is nothing to gain, or when part of
     * the scan has to run on this thread -- see the two gates inside.
     */
    prescan_map prescan_files(const std::vector<size_t>& file_order,
                              std::optional<ui_clock::time_point> deadline);

    /**
     * One progress tick for a fan-out over `work`: read each file's progress
     * slots into `in_flight` and hand the totals to lss_scan_progress.
     *
     * @return True if the callback asked for the scan to stop.
     */
    bool tick_scan_progress(const std::vector<logfile*>& work,
                            std::vector<index_progress_report>& in_flight);

    /** Re-run the filters over every file from where each one left off. */
    void reobserve_files();

    void clear_line_size_cache()
    {
        this->lss_line_size_cache.fill(std::make_pair(0, 0));
        this->lss_line_size_cache[0].first = -1;
    }

    bool check_extra_filters(iterator ld, logfile::iterator ll);

    size_t lss_basename_width = 0;
    size_t lss_filename_width = 0;
    line_context_t lss_line_context{line_context_t::none};
    bool lss_force_rebuild{false};
    bool lss_merge_deferred{false};
    std::vector<std::unique_ptr<logfile_data>> lss_files;
    unsigned int lss_all_timestamp_flags{0};

    std::vector<uint32_t> lss_filtered_index;

    // Summaries of fixed-size windows of `lss_filtered_index`, maintained by
    // update_regions().
    std::vector<index_region> lss_regions;
    // How many rows the summary table accounts for.  The trailing region is
    // usually partial, so its counts cannot be worked out from
    // lss_regions.size().
    size_t lss_regions_rows{0};

    /**
     * Bring `lss_regions` back into agreement with `lss_filtered_index`.
     *
     * @param valid_rows The number of rows at the front of the filtered index
     * that have not moved since the last call.  Regions covering anything
     * past that are dropped and recounted.
     */
    void update_regions(size_t valid_rows);

    void validate_regions() const;

    // Persistent state for incremental context expansion in rebuild_index()
    struct ctx_msg_range {
        size_t cmr_start{0};
        size_t cmr_count{0};
    };
    std::deque<ctx_msg_range> lss_ctx_before_msgs;
    size_t lss_ctx_after_msgs_remaining{0};
    bool lss_ctx_in_after{false};

    void flush_context_before_msgs();

    auto_mem<sqlite3_stmt> lss_preview_filter_stmt{sqlite3_finalize};

    std::map<std::string, breakpoint_info> lss_breakpoints;
    bookmarks<content_line_t>::type lss_user_marks{
        bookmarks<content_line_t>::create_array()};
    auto_mem<sqlite3_stmt> lss_marker_stmt{sqlite3_finalize};
    std::string lss_marker_stmt_text;

    line_flags_t lss_token_flags{0};
    iterator lss_token_file_data;
    std::shared_ptr<logfile> lss_token_file;
    attr_line_t lss_token_al;
    lnav::document::metadata lss_token_meta;
    int lss_token_meta_line{-1};
    int lss_token_meta_size{0};
    // the byte length of the time column that was prepended to the row that
    // was rendered last.  it is zeroed at the start of every render and only
    // refilled for a row that carries a timestamp.
    size_t lss_time_column_size{0};
    size_t lss_time_column_padding{0};
    // the display width of the time column.  unlike lss_time_column_size,
    // this is kept across renders so that callers outside of the render path
    // can find the indent without depending on which row came last.
    size_t lss_time_column_width{0};
    logline_value_vector lss_token_values;
    std::vector<std::pair<int, int>> lss_token_shifts;
    shared_buffer lss_share_manager;
    logfile::iterator lss_token_line;
    std::array<std::pair<int, size_t>, LINE_SIZE_CACHE_SIZE>
        lss_line_size_cache;
    bool lss_marked_only{false};
    index_delegate* lss_index_delegate{nullptr};
    scan_progress_fn lss_scan_progress;
    size_t lss_longest_line{0};
    meta_grepper lss_meta_grepper;
    log_location_history lss_location_history;
    exec_context* lss_exec_context{nullptr};

    bool lss_in_value_for_line{false};
    bool lss_line_meta_changed{false};

    bool lss_indexing_in_progress{false};
};

#endif