client_aic.models.core_result_ai.CoreResultAI

CoreResultAI

Source code in client_aic/models/core_result_ai.py
class CoreResultAI:
    """## CoreResultAI"""

    def __init__(
        self,
        id: int = None,
        user_id: int = None,
        job_id: int = None,
        worker_id: int = None,
        state: int = None,
        question: str = None,
        answer: str = None,
        model_name: str = None,
        score: float = None,
        question_score: str = None,
        answer_score: str = None,
        match_source: str = None,
        match_page: int = None,
        match_content: str = None,
        summarized_question: str = None,
        summarized_answer: str = None,
        summarized_score: float = None,
        reviewed_answer: str = None,
        reviewed_score: float = None,
        reviewed_computed_score: float = None,
        reviewed_notes: str = None,
        collection: str = None,
        collection_notes: str = None,
        session_id: str = None,
        derived_session_id: str = None,
        embed_model_name: str = None,
        category: str = None,
        tags: str = None,
        latency: float = None,
        data: dict = None,
        created_at: str = None,
        updated_at: str = None,
    ):
        """
        CoreResultAI

        ai results schema that supports,
        rag with RLHF (reinforcement learning
        with human feedback), summarization review,
        reviewing rag source confidence,
        historical context using session_ids

        the rest api processes
        ai jobs and an llm agent stores the results
        an CoreResultAI record.

        this class represents the
        underlying, public api model.

        :param id: db ai_result.id primary key
        :param user_id: user id
        :param job_id: job id
        :param worker_id: llm agent worker id
        :param state: result record state
        :param question: question for the llm
        :param answer: llm's answer to the
            questions
            :param model_name: model name (usually
            one from hugging face that is
            locally-stored like **falcon-170b** vs
            **mistral-7B**)
        :param score: llm's confidence in the
            answer with values between
            **0.00** <-> **1.00**
        :param question_score: similarity
            matching score for the llm's answer
            which shows how confident the
            llm is in the question from the rag
            data sources
        :param answer_score: similarity
            matching score for the llm's answer
            which shows how confident the
            llm is in the answer from the rag
            data sources
        :param match_source: best matching
            source/file on disk
            from the most confident rag data source
        :param match_page: best matching
            page extracted
            from the most confident rag data source
        :param match_content: best matching
            content extracted
            from the most confident rag data source
        :param summarized_question: after
            running the question, the autograder
            can provide a summarized question for
            reviewing historical trends and finding
            unique write-through cache keys
            within your llm ai results
        :param summarized_answer: after
            running the question, the autograder
            can provide a summarized answer for
            reviewing historical trends and finding
            unique write-through cache keys within
            your llm ai results
        :param summarized_score: after
            running the autograder
            summarizes the question and answer
            and provides a summarization score
            based off the rag data sources. this
            is helpful for tracking how many
            unique question/answer pairs
            does this llm need to support over time.
        :param reviewed_answer: subject matter
            expert's answer for this question
        :param reviewed_score: subject matter
            expert's confidence score for their
            **reviewed_answer** (score is on
            a similarity confidence scale with values
            betweeen
            **0.00** -> **1.00**)
        :param reviewed_computed_score: after
            grading the score, the auto-grader will
            recompute the score based off the current
            rag-sources in the pgvector embeddings
            and provided the best score from those
            data sources. if there is a low score, then
            the subject matter answer in the
            **reviewed_answer** field should be
            the preferred answer over the rag source(s).
            if this score is higher then there could
            be an issue with your lora/qlora peft
            configuration that is creating an output
            issue
        :param reviewed_notes: notes added during
            the human feedback update flow
        :param collection: name of the collection
            in an embedding pgvector database
            :param collection_notes: notes for this
        collection (e.g. useful for tracking collection id
            multi-tenant rag backend source matches with a
            data source location
            **s3_loc=s3://source_bucket**)
        :param session_id: custom session id
            for tracking the conversation history
            usually a **uuid** or
            **f'{user_id}.{job_type}.{uuid}'**
        :param derived_session_id: optional child
            session from the parent **session_id**
        :param embed_model_name: embedding model name
        :param category: custom label for
            this result
        :param tags: comma-delimited list of tags
        :param latency: latency for processing the
            job
        :param data: metadata dictionary
        :param created_at: utc timestamp creation date
        :param updated_at: utc timestamp last update
        """
        self.id = id
        self.user_id = user_id
        self.job_id = job_id
        self.worker_id = worker_id
        self.state = state
        self.question = question
        self.answer = answer
        self.model_name = model_name
        self.score = score
        self.question_score = question_score
        self.answer_score = answer_score
        self.match_source = match_source
        self.match_page = match_page
        self.match_content = match_content
        self.summarized_question = summarized_question
        self.summarized_answer = summarized_answer
        self.summarized_score = summarized_score
        self.reviewed_answer = reviewed_answer
        self.reviewed_score = reviewed_score
        self.reviewed_computed_score = (
            reviewed_computed_score
        )
        self.reviewed_notes = reviewed_notes
        self.collection = collection
        self.collection_notes = collection_notes
        self.session_id = session_id
        self.derived_session_id = derived_session_id
        self.embed_model_name = embed_model_name
        self.category = category
        self.tags = tags
        self.latency = latency
        self.data = data
        self.created_at = created_at
        self.updated_at = updated_at

    def load_response_dict(
        self,
        rec_dict,
    ):
        self.sql_query = rec_dict.get("sql_query", None)
        self.msg = rec_dict.get("msg", None)
        self.recs = []
        for rec in rec_dict.get("recs", []):
            self.add_rec(rec)
        self.id = rec_dict.get("id", None)
        self.user_id = rec_dict.get("user_id", None)
        self.job_id = rec_dict.get("job_id", None)
        self.worker_id = rec_dict.get("worker_id", None)
        self.state = rec_dict.get("state", None)
        self.question = rec_dict.get("question", None)
        self.answer = rec_dict.get("answer", None)
        self.model_name = rec_dict.get("model_name", None)
        self.score = rec_dict.get("score", None)
        self.question_score = rec_dict.get(
            "question_score", None
        )
        self.answer_score = rec_dict.get(
            "answer_score", None
        )
        self.match_source = rec_dict.get(
            "match_source", None
        )
        self.match_page = rec_dict.get("match_page", None)
        self.match_content = rec_dict.get(
            "match_content", None
        )
        self.summarized_question = rec_dict.get(
            "summarized_question", None
        )
        self.summarized_answer = rec_dict.get(
            "summarized_answer", None
        )
        self.summarized_score = rec_dict.get(
            "summarized_score", None
        )
        self.reviewed_answer = rec_dict.get(
            "reviewed_answer", None
        )
        self.reviewed_score = rec_dict.get(
            "reviewed_score", None
        )
        self.reviewed_computed_score = rec_dict.get(
            "reviewed_computed_score", None
        )
        self.reviewed_notes = rec_dict.get(
            "reviewed_notes", None
        )
        self.collection = rec_dict.get("collection", None)
        self.collection_notes = rec_dict.get(
            "collection_notes", None
        )
        self.session_id = rec_dict.get("session_id", None)
        self.derived_session_id = rec_dict.get(
            "derived_session_id", None
        )
        self.embed_model_name = rec_dict.get(
            "embed_model_name", None
        )
        self.category = rec_dict.get("category", None)
        self.tags = rec_dict.get("tags", None)
        self.latency = rec_dict.get("latency", None)
        self.data = rec_dict.get("data", None)
        self.created_at = rec_dict.get("created_at", None)
        self.updated_at = rec_dict.get("updated_at", None)

    def get_dict(self):
        """
        get_dict

        build a dictionary for the object and return
        the dictionary

        :returns: dictionary
        :rtype: dict
        """
        return {
            "id": self.id,
            "user_id": self.user_id,
            "job_id": self.job_id,
            "worker_id": self.worker_id,
            "state": self.state,
            "question": self.question,
            "answer": self.answer,
            "model_name": self.model_name,
            "score": self.score,
            "question_score": self.question_score,
            "answer_score": self.answer_score,
            "match_source": self.match_source,
            "match_page": self.match_page,
            "match_content": self.match_content,
            "summarized_question": self.summarized_question,
            "summarized_answer": self.summarized_answer,
            "summarized_score": self.summarized_score,
            "reviewed_answer": self.reviewed_answer,
            "reviewed_score": self.reviewed_score,
            "reviewed_computed_score": self.reviewed_computed_score,
            "reviewed_notes": self.reviewed_notes,
            "collection": self.collection,
            "collection_notes": self.collection_notes,
            "session_id": self.session_id,
            "derived_session_id": self.derived_session_id,
            "embed_model_name": self.embed_model_name,
            "category": self.category,
            "latency": self.latency,
            "tags": self.tags,
            "data": self.data,
            "created_at": self.created_at,
            "updated_at": self.updated_at,
        }

    def show(self):
        """
        show

        pretty-print as a
        dictionary to the log
        """
        log.info(
            f"CoreResultAI.id={self.id} values:\n"
            f"{ppj.ppj(self.get_dict())}"
        )

__init__(id=None, user_id=None, job_id=None, worker_id=None, state=None, question=None, answer=None, model_name=None, score=None, question_score=None, answer_score=None, match_source=None, match_page=None, match_content=None, summarized_question=None, summarized_answer=None, summarized_score=None, reviewed_answer=None, reviewed_score=None, reviewed_computed_score=None, reviewed_notes=None, collection=None, collection_notes=None, session_id=None, derived_session_id=None, embed_model_name=None, category=None, tags=None, latency=None, data=None, created_at=None, updated_at=None)

CoreResultAI

ai results schema that supports, rag with RLHF (reinforcement learning with human feedback), summarization review, reviewing rag source confidence, historical context using session_ids

the rest api processes ai jobs and an llm agent stores the results an CoreResultAI record.

this class represents the underlying, public api model.

Parameters:
  • id (int, default: None ) –

    db ai_result.id primary key

  • user_id (int, default: None ) –

    user id

  • job_id (int, default: None ) –

    job id

  • worker_id (int, default: None ) –

    llm agent worker id

  • state (int, default: None ) –

    result record state

  • question (str, default: None ) –

    question for the llm

  • answer (str, default: None ) –

    llm's answer to the questions :param model_name: model name (usually one from hugging face that is locally-stored like falcon-170b vs mistral-7B)

  • score (float, default: None ) –

    llm's confidence in the answer with values between 0.00 <-> 1.00

  • question_score (str, default: None ) –

    similarity matching score for the llm's answer which shows how confident the llm is in the question from the rag data sources

  • answer_score (str, default: None ) –

    similarity matching score for the llm's answer which shows how confident the llm is in the answer from the rag data sources

  • match_source (str, default: None ) –

    best matching source/file on disk from the most confident rag data source

  • match_page (int, default: None ) –

    best matching page extracted from the most confident rag data source

  • match_content (str, default: None ) –

    best matching content extracted from the most confident rag data source

  • summarized_question (str, default: None ) –

    after running the question, the autograder can provide a summarized question for reviewing historical trends and finding unique write-through cache keys within your llm ai results

  • summarized_answer (str, default: None ) –

    after running the question, the autograder can provide a summarized answer for reviewing historical trends and finding unique write-through cache keys within your llm ai results

  • summarized_score (float, default: None ) –

    after running the autograder summarizes the question and answer and provides a summarization score based off the rag data sources. this is helpful for tracking how many unique question/answer pairs does this llm need to support over time.

  • reviewed_answer (str, default: None ) –

    subject matter expert's answer for this question

  • reviewed_score (float, default: None ) –

    subject matter expert's confidence score for their reviewed_answer (score is on a similarity confidence scale with values betweeen 0.00 -> 1.00)

  • reviewed_computed_score (float, default: None ) –

    after grading the score, the auto-grader will recompute the score based off the current rag-sources in the pgvector embeddings and provided the best score from those data sources. if there is a low score, then the subject matter answer in the reviewed_answer field should be the preferred answer over the rag source(s). if this score is higher then there could be an issue with your lora/qlora peft configuration that is creating an output issue

  • reviewed_notes (str, default: None ) –

    notes added during the human feedback update flow

  • collection (str, default: None ) –

    name of the collection in an embedding pgvector database :param collection_notes: notes for this collection (e.g. useful for tracking collection id multi-tenant rag backend source matches with a data source location s3_loc=s3://source_bucket)

  • session_id (str, default: None ) –

    custom session id for tracking the conversation history usually a uuid or f'{user_id}.{job_type}.{uuid}'

  • derived_session_id (str, default: None ) –

    optional child session from the parent session_id

  • embed_model_name (str, default: None ) –

    embedding model name

  • category (str, default: None ) –

    custom label for this result

  • tags (str, default: None ) –

    comma-delimited list of tags

  • latency (float, default: None ) –

    latency for processing the job

  • data (dict, default: None ) –

    metadata dictionary

  • created_at (str, default: None ) –

    utc timestamp creation date

  • updated_at (str, default: None ) –

    utc timestamp last update

client_aic/models/core_result_ai.py
def __init__(
    self,
    id: int = None,
    user_id: int = None,
    job_id: int = None,
    worker_id: int = None,
    state: int = None,
    question: str = None,
    answer: str = None,
    model_name: str = None,
    score: float = None,
    question_score: str = None,
    answer_score: str = None,
    match_source: str = None,
    match_page: int = None,
    match_content: str = None,
    summarized_question: str = None,
    summarized_answer: str = None,
    summarized_score: float = None,
    reviewed_answer: str = None,
    reviewed_score: float = None,
    reviewed_computed_score: float = None,
    reviewed_notes: str = None,
    collection: str = None,
    collection_notes: str = None,
    session_id: str = None,
    derived_session_id: str = None,
    embed_model_name: str = None,
    category: str = None,
    tags: str = None,
    latency: float = None,
    data: dict = None,
    created_at: str = None,
    updated_at: str = None,
):
    """
    CoreResultAI

    ai results schema that supports,
    rag with RLHF (reinforcement learning
    with human feedback), summarization review,
    reviewing rag source confidence,
    historical context using session_ids

    the rest api processes
    ai jobs and an llm agent stores the results
    an CoreResultAI record.

    this class represents the
    underlying, public api model.

    :param id: db ai_result.id primary key
    :param user_id: user id
    :param job_id: job id
    :param worker_id: llm agent worker id
    :param state: result record state
    :param question: question for the llm
    :param answer: llm's answer to the
        questions
        :param model_name: model name (usually
        one from hugging face that is
        locally-stored like **falcon-170b** vs
        **mistral-7B**)
    :param score: llm's confidence in the
        answer with values between
        **0.00** <-> **1.00**
    :param question_score: similarity
        matching score for the llm's answer
        which shows how confident the
        llm is in the question from the rag
        data sources
    :param answer_score: similarity
        matching score for the llm's answer
        which shows how confident the
        llm is in the answer from the rag
        data sources
    :param match_source: best matching
        source/file on disk
        from the most confident rag data source
    :param match_page: best matching
        page extracted
        from the most confident rag data source
    :param match_content: best matching
        content extracted
        from the most confident rag data source
    :param summarized_question: after
        running the question, the autograder
        can provide a summarized question for
        reviewing historical trends and finding
        unique write-through cache keys
        within your llm ai results
    :param summarized_answer: after
        running the question, the autograder
        can provide a summarized answer for
        reviewing historical trends and finding
        unique write-through cache keys within
        your llm ai results
    :param summarized_score: after
        running the autograder
        summarizes the question and answer
        and provides a summarization score
        based off the rag data sources. this
        is helpful for tracking how many
        unique question/answer pairs
        does this llm need to support over time.
    :param reviewed_answer: subject matter
        expert's answer for this question
    :param reviewed_score: subject matter
        expert's confidence score for their
        **reviewed_answer** (score is on
        a similarity confidence scale with values
        betweeen
        **0.00** -> **1.00**)
    :param reviewed_computed_score: after
        grading the score, the auto-grader will
        recompute the score based off the current
        rag-sources in the pgvector embeddings
        and provided the best score from those
        data sources. if there is a low score, then
        the subject matter answer in the
        **reviewed_answer** field should be
        the preferred answer over the rag source(s).
        if this score is higher then there could
        be an issue with your lora/qlora peft
        configuration that is creating an output
        issue
    :param reviewed_notes: notes added during
        the human feedback update flow
    :param collection: name of the collection
        in an embedding pgvector database
        :param collection_notes: notes for this
    collection (e.g. useful for tracking collection id
        multi-tenant rag backend source matches with a
        data source location
        **s3_loc=s3://source_bucket**)
    :param session_id: custom session id
        for tracking the conversation history
        usually a **uuid** or
        **f'{user_id}.{job_type}.{uuid}'**
    :param derived_session_id: optional child
        session from the parent **session_id**
    :param embed_model_name: embedding model name
    :param category: custom label for
        this result
    :param tags: comma-delimited list of tags
    :param latency: latency for processing the
        job
    :param data: metadata dictionary
    :param created_at: utc timestamp creation date
    :param updated_at: utc timestamp last update
    """
    self.id = id
    self.user_id = user_id
    self.job_id = job_id
    self.worker_id = worker_id
    self.state = state
    self.question = question
    self.answer = answer
    self.model_name = model_name
    self.score = score
    self.question_score = question_score
    self.answer_score = answer_score
    self.match_source = match_source
    self.match_page = match_page
    self.match_content = match_content
    self.summarized_question = summarized_question
    self.summarized_answer = summarized_answer
    self.summarized_score = summarized_score
    self.reviewed_answer = reviewed_answer
    self.reviewed_score = reviewed_score
    self.reviewed_computed_score = (
        reviewed_computed_score
    )
    self.reviewed_notes = reviewed_notes
    self.collection = collection
    self.collection_notes = collection_notes
    self.session_id = session_id
    self.derived_session_id = derived_session_id
    self.embed_model_name = embed_model_name
    self.category = category
    self.tags = tags
    self.latency = latency
    self.data = data
    self.created_at = created_at
    self.updated_at = updated_at

get_dict()

get_dict

build a dictionary for the object and return the dictionary

Returns:
  • dict

    dictionary

client_aic/models/core_result_ai.py
def get_dict(self):
    """
    get_dict

    build a dictionary for the object and return
    the dictionary

    :returns: dictionary
    :rtype: dict
    """
    return {
        "id": self.id,
        "user_id": self.user_id,
        "job_id": self.job_id,
        "worker_id": self.worker_id,
        "state": self.state,
        "question": self.question,
        "answer": self.answer,
        "model_name": self.model_name,
        "score": self.score,
        "question_score": self.question_score,
        "answer_score": self.answer_score,
        "match_source": self.match_source,
        "match_page": self.match_page,
        "match_content": self.match_content,
        "summarized_question": self.summarized_question,
        "summarized_answer": self.summarized_answer,
        "summarized_score": self.summarized_score,
        "reviewed_answer": self.reviewed_answer,
        "reviewed_score": self.reviewed_score,
        "reviewed_computed_score": self.reviewed_computed_score,
        "reviewed_notes": self.reviewed_notes,
        "collection": self.collection,
        "collection_notes": self.collection_notes,
        "session_id": self.session_id,
        "derived_session_id": self.derived_session_id,
        "embed_model_name": self.embed_model_name,
        "category": self.category,
        "latency": self.latency,
        "tags": self.tags,
        "data": self.data,
        "created_at": self.created_at,
        "updated_at": self.updated_at,
    }

show()

show

pretty-print as a dictionary to the log

client_aic/models/core_result_ai.py
def show(self):
    """
    show

    pretty-print as a
    dictionary to the log
    """
    log.info(
        f"CoreResultAI.id={self.id} values:\n"
        f"{ppj.ppj(self.get_dict())}"
    )