{
  "label": "DEMO on public MAS documents. Not a client project. All numbers measured by rag_eval.py on the run date below.",
  "run_at": "2026-10-21T18:50:40+08:00",
  "environment": {
    "python": "3.13.5",
    "cpu": "x86_64",
    "torch_threads": 8,
    "device": "cpu",
    "index": "faiss IndexFlatIP (exact, cosine)"
  },
  "corpus": {
    "documents": [
      {
        "key": "guidelines",
        "title": "MAS Guidelines on AI Risk Management (final, 7 Oct 2026)",
        "url": "https://www.mas.gov.sg/-/media/mas-media-library/regulation/guidelines/bd/guidelines-on-artificial-intelligence-risk-management-for-financial-institutions/guidelines-on-artificial-intelligence-risk-management.pdf"
      },
      {
        "key": "consultation",
        "title": "MAS Consultation Paper P017-2025 on AI Risk Management (Nov 2025)",
        "url": "https://www.mas.gov.sg/-/media/mas-media-library/publications/consultations/bd/2025/final_consultation_paper_on_guidelines_on_ai_risk_management_forrelease.pdf"
      },
      {
        "key": "feat",
        "title": "MAS FEAT Principles (2018, updated Feb 2019)",
        "url": "https://www.mas.gov.sg/-/media/mas/news-and-publications/monographs-and-information-papers/feat-principles-updated-7-feb-19.pdf"
      }
    ],
    "n_chunks": 292,
    "chunk_words": 120,
    "overlap_words": 30
  },
  "metric_definitions": {
    "recall_at_5": "share of questions where a top-5 chunk overlaps the gold evidence span",
    "citation_hit_rate": "share of questions where the top-1 chunk is cited from the gold document AND page",
    "mrr_at_5": "mean reciprocal rank of first relevant chunk (0 if not in top 5)",
    "latency": "per-query wall time on CPU, each query repeated; 'embed+search' includes encoding the query"
  },
  "english_gold_set": {
    "sentence-transformers/all-MiniLM-L6-v2": {
      "model": "sentence-transformers/all-MiniLM-L6-v2",
      "dim": 384,
      "index_build_s": 2.88,
      "n_queries": 32,
      "recall_at_5": 0.844,
      "recall_at_1": 0.312,
      "citation_hit_rate": 0.312,
      "mrr_at_5": 0.519,
      "latency_ms": {
        "repeats_per_query": 5,
        "query_embed_plus_search_p50": 3.83,
        "query_embed_plus_search_p95": 29.65,
        "vector_search_only_p50": 0.039,
        "vector_search_only_p95": 0.052
      },
      "per_query": [
        {
          "id": "q01",
          "lang": "en",
          "question": "When do the MAS AI risk management guidelines take effect?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "7 October 2027; Sections 3-4 from 7 Oct 2027, Sections 5-6 by 7 Oct 2028 (para 1.8)."
          },
          "hit_at_5": true,
          "rank": 3,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 9,
            "score": 0.805,
            "snippet": "proportionate manner. General 4.6. In addition to the areas highlighted above, MAS welcomes comments on other aspects of the Guidelines. Question 9. MAS seeks comments on any aspects of the Guidelines that have not been …"
          }
        },
        {
          "id": "q02",
          "lang": "en",
          "question": "By when must FIs meet the AI life cycle controls and capability expectations?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "Sections 5 and 6 by 7 October 2028 (para 1.8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 16,
            "score": 0.656,
            "snippet": "Intelligence Risk Management 7 OCTOBER 2026 | 17 5. AI Life Cycle27 Controls 5.1. An FI should plan for and implement robust controls covering the entire life cycle of an AI use case, system or model and assign clear rol…"
          }
        },
        {
          "id": "q03",
          "lang": "en",
          "question": "Are rule-based calculators considered AI under the guidelines?",
          "gold": {
            "doc": "guidelines",
            "page": 4,
            "answer": "No: tools whose outputs are solely based on predefined programming logic or rules are not AI (para 1.3)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 6,
            "score": 0.639,
            "snippet": "manner are set out in paragraph 1.5 and the Annex of the Guidelines. Proposed AIRG | 7 may influence physical or virtual environments, and vary in their levels of autonomy and adaptiveness after deployment.14 Calculators…"
          }
        },
        {
          "id": "q04",
          "lang": "en",
          "question": "How do the guidelines define a model, a system and a use case?",
          "gold": {
            "doc": "guidelines",
            "page": 4,
            "answer": "Model converts assumptions and input data into outputs; system is one or more models plus other components; use case is a specific real-world context (para 1.4)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 4,
            "score": 0.628,
            "snippet": "include estimates, predictions, content, summaries, recommendations, or decisions that may influence physical or virtual environments. Different AI vary in their levels of autonomy and adaptiveness after deployment 6 . C…"
          }
        },
        {
          "id": "q05",
          "lang": "en",
          "question": "What reputational risk can customer-facing chatbots create?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "Chatbots could provide incorrect information or offensive remarks leading to negative media attention (para 1.9(e))."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 14,
            "score": 0.623,
            "snippet": "risks, e.g., unexpected behaviour of AI used to automate an FI’s operations could lead to operational disruptions or errors in critical processes. c. Conduct risk, e.g., use of AI can result in biased outputs, leading to…"
          }
        },
        {
          "id": "q06",
          "lang": "en",
          "question": "What security risks does Generative AI introduce, such as prompt injection?",
          "gold": {
            "doc": "guidelines",
            "page": 6,
            "answer": "Adversarial attacks via prompt injection or data poisoning (para 1.10(a))."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 5,
            "score": 0.748,
            "snippet": "over time. Guidelines on Artificial Intelligence Risk Management 7 OCTOBER 2026 | 6 area also make it harder to evaluate and test, as well as understand and explain Generative AI’s behaviour and outputs. The diverse and …"
          }
        },
        {
          "id": "q07",
          "lang": "en",
          "question": "What extra risks do AI agents with access to tools pose?",
          "gold": {
            "doc": "guidelines",
            "page": 6,
            "answer": "They could autonomously execute unauthorised or erroneous actions; compromised agents could exfiltrate data (para 1.11)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 15,
            "score": 0.713,
            "snippet": "mission critical areas, or skill degradation due to over-reliance on Generative AI. 1.10 Similarly, the use of newer technologies such as AI agents, which may be granted greater autonomy and access to tools, could furthe…"
          }
        },
        {
          "id": "q08",
          "lang": "en",
          "question": "When can a financial institution apply only basic AI governance policies?",
          "gold": {
            "doc": "guidelines",
            "page": 8,
            "answer": "If poor performance or unavailability of the AI is unlikely to have a material adverse impact on the FI, customers or stakeholders (para 2.3)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 3,
            "score": 0.658,
            "snippet": "seek to support their responsible adoption of AI. These Guidelines set out MAS’ supervisory expectations in the areas of oversight, key AI risk management systems, policies, procedures, AI life cycle controls, and capabi…"
          }
        },
        {
          "id": "q09",
          "lang": "en",
          "question": "What must basic AI governance policies minimally include?",
          "gold": {
            "doc": "guidelines",
            "page": 9,
            "answer": "Clear accountability for AI oversight, permitted and prohibited uses with human oversight, etc. (para 2.5)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 9,
            "score": 0.679,
            "snippet": "to assist in designing marketing or internal materials; and (f) Using AI-powered chatbots to assist personnel in locating relevant internal resources, such as policies or procedures. 2.5. Basic AI governance policies and…"
          }
        },
        {
          "id": "q10",
          "lang": "en",
          "question": "What is the board responsible for regarding AI risk management?",
          "gold": {
            "doc": "guidelines",
            "page": 10,
            "answer": "Approving and reviewing the governance approach, ensuring AI risks are addressed in the risk appetite framework, etc. (para 3.4)."
          },
          "hit_at_5": true,
          "rank": 3,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 16,
            "score": 0.839,
            "snippet": "and managed by existing risk management functions, the Board and senior management should still ensure consistent and coordinated management of AI risks. Where overall AI risk exposure of the FI is deemed material12, the…"
          }
        },
        {
          "id": "q11",
          "lang": "en",
          "question": "What should an FI do about residual risk from unidentified or shadow AI use?",
          "gold": {
            "doc": "guidelines",
            "page": 13,
            "answer": "Ensure residual risks stay within risk appetite; identify risks from constraints and put mitigants in place (para 4.4)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 13,
            "score": 0.695,
            "snippet": "the FI's risk appetite. Where there are practical constraints in consistently identifying all AI use, e.g., due to inadequate 21 For example, software-as-a-service which may not be explicitly sold as AI, but contains AI …"
          }
        },
        {
          "id": "q12",
          "lang": "en",
          "question": "Do FIs need to keep an inventory of AI use cases?",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Yes, an accurate inventory of AI use cases, systems or models across the FI (para 4.5)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 4,
            "score": 0.707,
            "snippet": "or more models and other machine-based components. (c) A use case refers to a specific real-world context that the model or system is applied to. 1.5. The Guidelines establish a set of high-level expectations that all FI…"
          }
        },
        {
          "id": "q13",
          "lang": "en",
          "question": "Which attributes should an AI inventory record?",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Purpose, approved scope, model type, data used, dependencies, lifecycle status, risk materiality rating, review status, roles, documentation links (para 4.7)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 14,
            "score": 0.709,
            "snippet": "models or establish a dedicated AI inventory. In either case, there should be clear linkages between the AI inventory and other relevant inventories23 in the FI, to facilitate cross-referencing between inventories as par…"
          }
        },
        {
          "id": "q14",
          "lang": "en",
          "question": "Who should oversee the AI inventory to keep it accurate and complete?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "A designated control function with independent oversight (para 4.9)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 14,
            "score": 0.748,
            "snippet": "the AI inventory should not compromise the effectiveness of the FI’s oversight and risk management of AI. Guidelines on Artificial Intelligence Risk Management 7 OCTOBER 2026 | 15 4.8. The design of the inventory should …"
          }
        },
        {
          "id": "q15",
          "lang": "en",
          "question": "What is the difference between inherent and residual risk materiality?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Inherent is before controls, residual after; residual must meet risk appetite before deployment (para 4.11)."
          },
          "hit_at_5": true,
          "rank": 4,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 15,
            "score": 0.528,
            "snippet": "management controls are applied, as well as the residual risk materiality after risk management controls are applied. The FI should ensure that the residual risk materiality of the AI use case meets the FI’s risk appetit…"
          }
        },
        {
          "id": "q16",
          "lang": "en",
          "question": "What three dimensions must a risk materiality assessment cover?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Impact, complexity and reliance (para 4.12)."
          },
          "hit_at_5": true,
          "rank": 3,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 18,
            "score": 0.606,
            "snippet": "that AI usage remains within approved boundaries. 3.9 The assessment should take into account both the inherent risk materiality of an AI use case, system or model before the appropriate risk management controls are appl…"
          }
        },
        {
          "id": "q17",
          "lang": "en",
          "question": "What does the reliance dimension consider?",
          "gold": {
            "doc": "guidelines",
            "page": 16,
            "answer": "Level of reliance on AI, autonomy granted and degree of human involvement (para 4.12(c))."
          },
          "hit_at_5": true,
          "rank": 4,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 18,
            "score": 0.231,
            "snippet": "of the AI technology used, the novelty of its application, or the data it uses. As the understanding of AI technologies evolves across time, this risk dimension could also evolve, e.g., with more research and greater fam…"
          }
        },
        {
          "id": "q18",
          "lang": "en",
          "question": "What contingency plans are needed for high-risk AI, including kill switches?",
          "gold": {
            "doc": "guidelines",
            "page": 17,
            "answer": "Fallback options such as alternative systems or manual processes; tested kill-switch activation protocols (para 5.3)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 17,
            "score": 0.631,
            "snippet": "and end-users, close monitoring of usage patterns and outputs for anomalies and ensuring compliance with the limited scope of use. 29 “Kill switches” are usually used to refer to mechanisms that can be used to deactivate…"
          }
        },
        {
          "id": "q19",
          "lang": "en",
          "question": "What data management controls are expected for data used in AI?",
          "gold": {
            "doc": "guidelines",
            "page": 17,
            "answer": "Data fit for purpose, representative, high quality and under robust data governance (para 5.4)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 19,
            "score": 0.675,
            "snippet": "data management controls to ensure data used across the AI life cycle is fit for purpose and representative, of high quality, and subject to robust data governance. General data governance and management standards, in ar…"
          }
        },
        {
          "id": "q20",
          "lang": "en",
          "question": "How should human oversight account for automation bias?",
          "gold": {
            "doc": "guidelines",
            "page": 20,
            "answer": "Oversight proportionate to risk, taking into account automation bias and decision fatigue (para 5.9)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 20,
            "score": 0.737,
            "snippet": "disadvantages for specific groups using appropriate fairness metrics, and documenting the results and any mitigation steps taken. Human Oversight 5.9. An FI should put in place and regularly review controls to ensure app…"
          }
        },
        {
          "id": "q21",
          "lang": "en",
          "question": "Does evaluation and testing require clear measurable thresholds before deployment?",
          "gold": {
            "doc": "guidelines",
            "page": 23,
            "answer": "Yes, identify key risks and set clear, measurable thresholds, proportionate to risk materiality (para 5.14)."
          },
          "hit_at_5": true,
          "rank": 5,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 25,
            "score": 0.487,
            "snippet": "selection rationale; (c) training procedures including code versions, environments, and hyperparameters; (d) evaluation measures and performance thresholds, testing approaches, and results; (e) explainability analysis, f…"
          }
        },
        {
          "id": "q22",
          "lang": "en",
          "question": "How detailed must AI development documentation be for auditability?",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Sufficient for an independent reviewer or auditor to understand and potentially replicate it (para 5.17)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 25,
            "score": 0.825,
            "snippet": "Reproducibility and Auditability 5.17. An FI should document the AI development process to enable reproducibility and auditability. The extent and rigour of documentation should be commensurate with the risk materiality …"
          }
        },
        {
          "id": "q23",
          "lang": "en",
          "question": "Who should review an AI use case before deployment?",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Parties not involved in its development (para 5.18)."
          },
          "hit_at_5": true,
          "rank": 5,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 24,
            "score": 0.71,
            "snippet": "code versions, environments, and hyperparameters; d. evaluation measures and performance thresholds, testing approaches, and results; e. explainability analysis, fairness assessments; and f. key assumptions, limitations …"
          }
        },
        {
          "id": "q24",
          "lang": "en",
          "question": "Why is ongoing monitoring of deployed AI needed?",
          "gold": {
            "doc": "guidelines",
            "page": 27,
            "answer": "Model staleness and performance degradation from data or model drift (para 5.23)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 25,
            "score": 0.708,
            "snippet": "and robust controls for the ongoing monitoring of all deployed AI (including third-party AI used in the FI). Given the uncertainties associated with AI, their dynamic nature and the potential for model staleness and perf…"
          }
        },
        {
          "id": "q25",
          "lang": "en",
          "question": "What should controls for decommissioning AI consider?",
          "gold": {
            "doc": "guidelines",
            "page": 17,
            "answer": "Dependencies, data retention, secure removal from production, stakeholder notifications (para 5.26)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 26,
            "score": 0.614,
            "snippet": "or system, depending on the severity and nature of the issue. For high risk materiality AI, consider implementing \"kill switches\" or override mechanisms to rapidly deactivate the AI system if it exceeds risk tolerances. …"
          }
        },
        {
          "id": "q26",
          "lang": "en",
          "question": "Does the guideline mention GPUs and infrastructure adequacy?",
          "gold": {
            "doc": "guidelines",
            "page": 30,
            "answer": "Yes, hardware such as GPUs, network, memory and secure data pipelines must be sufficient (para 6.3)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 27,
            "score": 0.461,
            "snippet": "these risks, the FI should ensure that the underlying hardware and software resources, such as graphics processing units, network infrastructure, system memory, secure data pipelines are sufficient to meet performance, s…"
          }
        },
        {
          "id": "q27",
          "lang": "en",
          "question": "When did the consultation on the AI risk management guidelines close?",
          "gold": {
            "doc": "consultation",
            "page": 3,
            "answer": "31 January 2026 (CP para 1.5)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 10,
            "score": 0.77,
            "snippet": "controls that should be applied across the entire AI life cycle, and 9 List of Questions | 11 the key areas that FIs should assess for relevance to the AI model, system or use case, and apply in a proportionate manner. Q…"
          }
        },
        {
          "id": "q28",
          "lang": "en",
          "question": "What is Project MindForge?",
          "gold": {
            "doc": "consultation",
            "page": 4,
            "answer": "Established to examine Generative AI risks and opportunities; consortium released a GenAI risk framework and works on an AI Risk Management Handbook (CP para 2.5)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "consultation",
            "page": 4,
            "score": 0.501,
            "snippet": "methodologies, a toolkit, and accompanying case studies. 2.5. With the emergence of Generative AI, Project MindForge4 was established to examine the risks and opportunities of Generative AI. The first phase of Project Mi…"
          }
        },
        {
          "id": "q29",
          "lang": "en",
          "question": "What is the Veritas Initiative?",
          "gold": {
            "doc": "consultation",
            "page": 4,
            "answer": "Industry consortium from Nov 2019 helping FIs incorporate FEAT principles; released methodologies and toolkit (CP para 2.4)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "consultation",
            "page": 4,
            "score": 0.522,
            "snippet": "MAS had established key principles to guide FIs in their responsible use of AI. In 2018, MAS co-created the principles of Fairness, Ethics, Accountability and Transparency (FEAT)2 with the financial industry to promote t…"
          }
        },
        {
          "id": "q30",
          "lang": "en",
          "question": "What does FEAT stand for and what are its accuracy and bias principles?",
          "gold": {
            "doc": "feat",
            "page": 6,
            "answer": "Fairness, Ethics, Accountability, Transparency; data and models regularly reviewed and validated for accuracy and to minimise bias (FEAT principles 3-4)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 20,
            "score": 0.309,
            "snippet": "use of AI increases in speed and scale. Key areas that the FI should consider include: (a) Roles and Responsibilities: Clear assignment of roles and responsibilities for human oversight, including escalation and decision…"
          }
        },
        {
          "id": "q31",
          "lang": "en",
          "question": "Can customers appeal or request review of AI-driven decisions under FEAT?",
          "gold": {
            "doc": "feat",
            "page": 6,
            "answer": "Yes, data subjects are provided channels to enquire, appeal and request reviews (FEAT principle 10)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 20,
            "score": 0.518,
            "snippet": "use of AI increases in speed and scale. Key areas that the FI should consider include: (a) Roles and Responsibilities: Clear assignment of roles and responsibilities for human oversight, including escalation and decision…"
          }
        },
        {
          "id": "q32",
          "lang": "en",
          "question": "Are firms accountable for externally sourced AI models under FEAT?",
          "gold": {
            "doc": "feat",
            "page": 6,
            "answer": "Yes, accountable for both internally developed and externally sourced models (FEAT principle 8)."
          },
          "hit_at_5": true,
          "rank": 5,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 14,
            "score": 0.524,
            "snippet": "uplifting technical controls (e.g., network monitoring, data loss prevention) to detect unauthorised use of AI. 23 Such linkages would depend on the attributes captured in the AI inventory. For example, the AI inventory …"
          }
        }
      ]
    },
    "BAAI/bge-small-en-v1.5": {
      "model": "BAAI/bge-small-en-v1.5",
      "dim": 384,
      "index_build_s": 6.35,
      "n_queries": 32,
      "recall_at_5": 0.781,
      "recall_at_1": 0.375,
      "citation_hit_rate": 0.344,
      "mrr_at_5": 0.531,
      "latency_ms": {
        "repeats_per_query": 5,
        "query_embed_plus_search_p50": 8.61,
        "query_embed_plus_search_p95": 67.92,
        "vector_search_only_p50": 0.039,
        "vector_search_only_p95": 0.074
      },
      "per_query": [
        {
          "id": "q01",
          "lang": "en",
          "question": "When do the MAS AI risk management guidelines take effect?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "7 October 2027; Sections 3-4 from 7 Oct 2027, Sections 5-6 by 7 Oct 2028 (para 1.8)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 4,
            "score": 0.825,
            "snippet": "fixed programmed instructions without learning or adaptation would not ordinarily be considered AI for the purpose of these Guidelines. 7 Where a point pertains specifically to an AI use case, AI system or AI model, we w…"
          }
        },
        {
          "id": "q02",
          "lang": "en",
          "question": "By when must FIs meet the AI life cycle controls and capability expectations?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "Sections 5 and 6 by 7 October 2028 (para 1.8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 8,
            "score": 0.786,
            "snippet": "on the proposed risk dimensions of impact, complexity and reliance that should be captured by FIs in AI risk materiality assessments, and whether there are any other risk dimensions that should be included. |9 AI Life Cy…"
          }
        },
        {
          "id": "q03",
          "lang": "en",
          "question": "Are rule-based calculators considered AI under the guidelines?",
          "gold": {
            "doc": "guidelines",
            "page": 4,
            "answer": "No: tools whose outputs are solely based on predefined programming logic or rules are not AI (para 1.3)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 6,
            "score": 0.783,
            "snippet": "manner are set out in paragraph 1.5 and the Annex of the Guidelines. Proposed AIRG | 7 may influence physical or virtual environments, and vary in their levels of autonomy and adaptiveness after deployment.14 Calculators…"
          }
        },
        {
          "id": "q04",
          "lang": "en",
          "question": "How do the guidelines define a model, a system and a use case?",
          "gold": {
            "doc": "guidelines",
            "page": 4,
            "answer": "Model converts assumptions and input data into outputs; system is one or more models plus other components; use case is a specific real-world context (para 1.4)."
          },
          "hit_at_5": true,
          "rank": 3,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 4,
            "score": 0.819,
            "snippet": "or more models and other machine-based components. (c) A use case refers to a specific real-world context that the model or system is applied to. 1.5. The Guidelines establish a set of high-level expectations that all FI…"
          }
        },
        {
          "id": "q05",
          "lang": "en",
          "question": "What reputational risk can customer-facing chatbots create?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "Chatbots could provide incorrect information or offensive remarks leading to negative media attention (para 1.9(e))."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 5,
            "score": 0.757,
            "snippet": "AI can result in biased outputs, leading to unfair treatment of certain customer groups, a bias for certain investment types over others, or conflicts of interests between an FI and its customers. (d) Financial crime ris…"
          }
        },
        {
          "id": "q06",
          "lang": "en",
          "question": "What security risks does Generative AI introduce, such as prompt injection?",
          "gold": {
            "doc": "guidelines",
            "page": 6,
            "answer": "Adversarial attacks via prompt injection or data poisoning (para 1.10(a))."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 4,
            "score": 0.828,
            "snippet": "or financial loss to customers. 2.2. Generative AI also introduces risks that are less well understood and harder to mitigate, such as hallucinations that produce convincing but false information, unpredictable behaviour…"
          }
        },
        {
          "id": "q07",
          "lang": "en",
          "question": "What extra risks do AI agents with access to tools pose?",
          "gold": {
            "doc": "guidelines",
            "page": 6,
            "answer": "They could autonomously execute unauthorised or erroneous actions; compromised agents could exfiltrate data (para 1.11)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 6,
            "score": 0.778,
            "snippet": "poor security controls. (e) Operational risks, e.g., potential service disruptions due to Generative AI outages and problems arising from inadequate pre-deployment testing. (f) Human-factor risks, e.g., use of shadow AI,…"
          }
        },
        {
          "id": "q08",
          "lang": "en",
          "question": "When can a financial institution apply only basic AI governance policies?",
          "gold": {
            "doc": "guidelines",
            "page": 8,
            "answer": "If poor performance or unavailability of the AI is unlikely to have a material adverse impact on the FI, customers or stakeholders (para 2.3)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 9,
            "score": 0.729,
            "snippet": "to assist in designing marketing or internal materials; and (f) Using AI-powered chatbots to assist personnel in locating relevant internal resources, such as policies or procedures. 2.5. Basic AI governance policies and…"
          }
        },
        {
          "id": "q09",
          "lang": "en",
          "question": "What must basic AI governance policies minimally include?",
          "gold": {
            "doc": "guidelines",
            "page": 9,
            "answer": "Clear accountability for AI oversight, permitted and prohibited uses with human oversight, etc. (para 2.5)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 9,
            "score": 0.794,
            "snippet": "to assist in designing marketing or internal materials; and (f) Using AI-powered chatbots to assist personnel in locating relevant internal resources, such as policies or procedures. 2.5. Basic AI governance policies and…"
          }
        },
        {
          "id": "q10",
          "lang": "en",
          "question": "What is the board responsible for regarding AI risk management?",
          "gold": {
            "doc": "guidelines",
            "page": 10,
            "answer": "Approving and reviewing the governance approach, ensuring AI risks are addressed in the risk appetite framework, etc. (para 3.4)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 10,
            "score": 0.79,
            "snippet": "board and senior management should ensure consistent standards, clear accountability, and robust coordination across the FI to manage risks from AI use. While the FI may adopt varying approaches to AI risk management, su…"
          }
        },
        {
          "id": "q11",
          "lang": "en",
          "question": "What should an FI do about residual risk from unidentified or shadow AI use?",
          "gold": {
            "doc": "guidelines",
            "page": 13,
            "answer": "Ensure residual risks stay within risk appetite; identify risks from constraints and put mitigants in place (para 4.4)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 14,
            "score": 0.766,
            "snippet": "risks, e.g., unexpected behaviour of AI used to automate an FI’s operations could lead to operational disruptions or errors in critical processes. c. Conduct risk, e.g., use of AI can result in biased outputs, leading to…"
          }
        },
        {
          "id": "q12",
          "lang": "en",
          "question": "Do FIs need to keep an inventory of AI use cases?",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Yes, an accurate inventory of AI use cases, systems or models across the FI (para 4.5)."
          },
          "hit_at_5": true,
          "rank": 3,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 10,
            "score": 0.793,
            "snippet": "systems, to facilitate the consistent identification of AI across all relevant business and functional areas. 8 Question 6 MAS seeks comments on the proposal for FIs to establish and maintain an accurate and up-to-date i…"
          }
        },
        {
          "id": "q13",
          "lang": "en",
          "question": "Which attributes should an AI inventory record?",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Purpose, approved scope, model type, data used, dependencies, lifecycle status, risk materiality rating, review status, roles, documentation links (para 4.7)."
          },
          "hit_at_5": true,
          "rank": 4,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 17,
            "score": 0.79,
            "snippet": "maintenance of the inventory, with new, updated or decommissioned AI use cases, systems or models reflected accurately. The FI can enhance existing inventories to include AI use cases, systems or models or establish a de…"
          }
        },
        {
          "id": "q14",
          "lang": "en",
          "question": "Who should oversee the AI inventory to keep it accurate and complete?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "A designated control function with independent oversight (para 4.9)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 14,
            "score": 0.755,
            "snippet": "the AI inventory should not compromise the effectiveness of the FI’s oversight and risk management of AI. Guidelines on Artificial Intelligence Risk Management 7 OCTOBER 2026 | 15 4.8. The design of the inventory should …"
          }
        },
        {
          "id": "q15",
          "lang": "en",
          "question": "What is the difference between inherent and residual risk materiality?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Inherent is before controls, residual after; residual must meet risk appetite before deployment (para 4.11)."
          },
          "hit_at_5": true,
          "rank": 3,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 18,
            "score": 0.743,
            "snippet": "that AI usage remains within approved boundaries. 3.9 The assessment should take into account both the inherent risk materiality of an AI use case, system or model before the appropriate risk management controls are appl…"
          }
        },
        {
          "id": "q16",
          "lang": "en",
          "question": "What three dimensions must a risk materiality assessment cover?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Impact, complexity and reliance (para 4.12)."
          },
          "hit_at_5": true,
          "rank": 3,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 18,
            "score": 0.805,
            "snippet": "that AI usage remains within approved boundaries. 3.9 The assessment should take into account both the inherent risk materiality of an AI use case, system or model before the appropriate risk management controls are appl…"
          }
        },
        {
          "id": "q17",
          "lang": "en",
          "question": "What does the reliance dimension consider?",
          "gold": {
            "doc": "guidelines",
            "page": 16,
            "answer": "Level of reliance on AI, autonomy granted and degree of human involvement (para 4.12(c))."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 16,
            "score": 0.658,
            "snippet": "that was not as well understood initially may change. For third-party AI, the level of visibility into the nature of the AI technology or the data used should also be considered. (c) Reliance: The level of reliance on AI…"
          }
        },
        {
          "id": "q18",
          "lang": "en",
          "question": "What contingency plans are needed for high-risk AI, including kill switches?",
          "gold": {
            "doc": "guidelines",
            "page": 17,
            "answer": "Fallback options such as alternative systems or manual processes; tested kill-switch activation protocols (para 5.3)."
          },
          "hit_at_5": true,
          "rank": 5,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 19,
            "score": 0.771,
            "snippet": "residual risks remain within the risk appetite of the FI. 4.4 For an AI use case, system or model assessed as high risk, an FI should develop and implement contingency plans. These plans should outline fallback options, …"
          }
        },
        {
          "id": "q19",
          "lang": "en",
          "question": "What data management controls are expected for data used in AI?",
          "gold": {
            "doc": "guidelines",
            "page": 17,
            "answer": "Data fit for purpose, representative, high quality and under robust data governance (para 5.4)."
          },
          "hit_at_5": true,
          "rank": 5,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 18,
            "score": 0.774,
            "snippet": "for anomalies, drifts, and potential bias. (d) Data Classification: Appropriate data classification processes to guide appropriate use of data in an AI use case, system or model, taking into account criticality and sensi…"
          }
        },
        {
          "id": "q20",
          "lang": "en",
          "question": "How should human oversight account for automation bias?",
          "gold": {
            "doc": "guidelines",
            "page": 20,
            "answer": "Oversight proportionate to risk, taking into account automation bias and decision fatigue (para 5.9)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 21,
            "score": 0.795,
            "snippet": "across its life cycle. The need for and degree of human oversight should take into account the objective of using AI, and be proportionate to the risk materiality of AI used. It should also take into account automation b…"
          }
        },
        {
          "id": "q21",
          "lang": "en",
          "question": "Does evaluation and testing require clear measurable thresholds before deployment?",
          "gold": {
            "doc": "guidelines",
            "page": 23,
            "answer": "Yes, identify key risks and set clear, measurable thresholds, proportionate to risk materiality (para 5.14)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 23,
            "score": 0.748,
            "snippet": "risk materiality of the applicable AI use case before deployment. An FI should identify key risks arising from the AI use case and set clear, measurable thresholds. Identification of risks and reliability and safety asse…"
          }
        },
        {
          "id": "q22",
          "lang": "en",
          "question": "How detailed must AI development documentation be for auditability?",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Sufficient for an independent reviewer or auditor to understand and potentially replicate it (para 5.17)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 24,
            "score": 0.857,
            "snippet": "compliance or operational risks, such as cybersecurity issues or service disruptions at third-party providers. Reproducibility & Auditability 4.17 An FI should document the AI development process to enable reproducibilit…"
          }
        },
        {
          "id": "q23",
          "lang": "en",
          "question": "Who should review an AI use case before deployment?",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Parties not involved in its development (para 5.18)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 24,
            "score": 0.825,
            "snippet": "code versions, environments, and hyperparameters; d. evaluation measures and performance thresholds, testing approaches, and results; e. explainability analysis, fairness assessments; and f. key assumptions, limitations …"
          }
        },
        {
          "id": "q24",
          "lang": "en",
          "question": "Why is ongoing monitoring of deployed AI needed?",
          "gold": {
            "doc": "guidelines",
            "page": 27,
            "answer": "Model staleness and performance degradation from data or model drift (para 5.23)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 27,
            "score": 0.842,
            "snippet": "and implement safeguards, such as input validation, throttling and anomaly detection. Post-Deployment Monitoring and Review 5.23. An FI should develop and implement comprehensive and robust controls for the ongoing monit…"
          }
        },
        {
          "id": "q25",
          "lang": "en",
          "question": "What should controls for decommissioning AI consider?",
          "gold": {
            "doc": "guidelines",
            "page": 17,
            "answer": "Dependencies, data retention, secure removal from production, stakeholder notifications (para 5.26)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 27,
            "score": 0.752,
            "snippet": "enhanced data quality checks, and more stringent performance monitoring. 4.26 An FI should develop and implement clear controls for the eventual retirement or decommissioning of AI when they are no longer needed or excee…"
          }
        },
        {
          "id": "q26",
          "lang": "en",
          "question": "Does the guideline mention GPUs and infrastructure adequacy?",
          "gold": {
            "doc": "guidelines",
            "page": 30,
            "answer": "Yes, hardware such as GPUs, network, memory and secure data pipelines must be sufficient (para 6.3)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 30,
            "score": 0.712,
            "snippet": "should ensure that the underlying hardware and software resources, such as graphics processing units, network infrastructure, system memory, secure data pipelines are sufficient to meet performance, scalability, and resi…"
          }
        },
        {
          "id": "q27",
          "lang": "en",
          "question": "When did the consultation on the AI risk management guidelines close?",
          "gold": {
            "doc": "consultation",
            "page": 3,
            "answer": "31 January 2026 (CP para 1.5)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 1,
            "score": 0.778,
            "snippet": "Consultation Paper P017-2025 – November 2025 Consultation Paper on Guidelines on Artificial Intelligence Risk Management Preface | 2 Contents 1. Preface 3 2. MAS’ Supervisory Approach to AI Risk Management 4 3. Applicabi…"
          }
        },
        {
          "id": "q28",
          "lang": "en",
          "question": "What is Project MindForge?",
          "gold": {
            "doc": "consultation",
            "page": 4,
            "answer": "Established to examine Generative AI risks and opportunities; consortium released a GenAI risk framework and works on an AI Risk Management Handbook (CP para 2.5)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "consultation",
            "page": 4,
            "score": 0.655,
            "snippet": "methodologies, a toolkit, and accompanying case studies. 2.5. With the emergence of Generative AI, Project MindForge4 was established to examine the risks and opportunities of Generative AI. The first phase of Project Mi…"
          }
        },
        {
          "id": "q29",
          "lang": "en",
          "question": "What is the Veritas Initiative?",
          "gold": {
            "doc": "consultation",
            "page": 4,
            "answer": "Industry consortium from Nov 2019 helping FIs incorporate FEAT principles; released methodologies and toolkit (CP para 2.4)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "consultation",
            "page": 4,
            "score": 0.643,
            "snippet": "MAS had established key principles to guide FIs in their responsible use of AI. In 2018, MAS co-created the principles of Fairness, Ethics, Accountability and Transparency (FEAT)2 with the financial industry to promote t…"
          }
        },
        {
          "id": "q30",
          "lang": "en",
          "question": "What does FEAT stand for and what are its accuracy and bias principles?",
          "gold": {
            "doc": "feat",
            "page": 6,
            "answer": "Fairness, Ethics, Accountability, Transparency; data and models regularly reviewed and validated for accuracy and to minimise bias (FEAT principles 3-4)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 1,
            "score": 0.624,
            "snippet": "Principles to Promote Fairness, Ethics, Accountability and Transparency (FEAT) in the Use of Artificial Intelligence and Data Analytics in Singapore’s Financial Sector Monetary Authority Of Singapore 1 Principles to Prom…"
          }
        },
        {
          "id": "q31",
          "lang": "en",
          "question": "Can customers appeal or request review of AI-driven decisions under FEAT?",
          "gold": {
            "doc": "feat",
            "page": 6,
            "answer": "Yes, data subjects are provided channels to enquire, appeal and request reviews (FEAT principle 10)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 19,
            "score": 0.706,
            "snippet": "with high impact on customer outcomes (e.g., provision of financial advisory services or fund management), will require a higher level of transparency and explainability than AI used solely to assist internal users in lo…"
          }
        },
        {
          "id": "q32",
          "lang": "en",
          "question": "Are firms accountable for externally sourced AI models under FEAT?",
          "gold": {
            "doc": "feat",
            "page": 6,
            "answer": "Yes, accountable for both internally developed and externally sourced models (FEAT principle 8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 22,
            "score": 0.756,
            "snippet": "oversight, or making the appropriate disclosures to users. b. Fairness: Exercise due diligence regarding the fairness practices of third-party AI providers, as the FI remains accountable for the fairness outcomes of thir…"
          }
        }
      ]
    },
    "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2": {
      "model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
      "dim": 384,
      "index_build_s": 3.87,
      "n_queries": 32,
      "recall_at_5": 0.531,
      "recall_at_1": 0.25,
      "citation_hit_rate": 0.344,
      "mrr_at_5": 0.367,
      "latency_ms": {
        "repeats_per_query": 5,
        "query_embed_plus_search_p50": 7.28,
        "query_embed_plus_search_p95": 76.16,
        "vector_search_only_p50": 0.04,
        "vector_search_only_p95": 0.067
      },
      "per_query": [
        {
          "id": "q01",
          "lang": "en",
          "question": "When do the MAS AI risk management guidelines take effect?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "7 October 2027; Sections 3-4 from 7 Oct 2027, Sections 5-6 by 7 Oct 2028 (para 1.8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 3,
            "score": 0.833,
            "snippet": "and set out MAS’ supervisory expectations relating to AI risk management in the financial sector. The Guidelines focus on oversight of AI risk management in FIs, key AI risk management systems, policies and procedures, k…"
          }
        },
        {
          "id": "q02",
          "lang": "en",
          "question": "By when must FIs meet the AI life cycle controls and capability expectations?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "Sections 5 and 6 by 7 October 2028 (para 1.8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 19,
            "score": 0.835,
            "snippet": "the AI life cycle when new information is available. 4.3 An FI may implement the relevant AI lifecycle controls in a proportionate manner, ensuring that they are appropriate to the assessed risk materiality of AI14. Wher…"
          }
        },
        {
          "id": "q03",
          "lang": "en",
          "question": "Are rule-based calculators considered AI under the guidelines?",
          "gold": {
            "doc": "guidelines",
            "page": 4,
            "answer": "No: tools whose outputs are solely based on predefined programming logic or rules are not AI (para 1.3)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 8,
            "score": 0.697,
            "snippet": "Such controls may be calibrated based on their relevance12 to the AI use cases, systems or models in the FI. Where such aspects are relevant to the AI use case, system or model, the FI may implement them based on the ris…"
          }
        },
        {
          "id": "q04",
          "lang": "en",
          "question": "How do the guidelines define a model, a system and a use case?",
          "gold": {
            "doc": "guidelines",
            "page": 4,
            "answer": "Model converts assumptions and input data into outputs; system is one or more models plus other components; use case is a specific real-world context (para 1.4)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 4,
            "score": 0.78,
            "snippet": "or more models and other machine-based components. (c) A use case refers to a specific real-world context that the model or system is applied to. 1.5. The Guidelines establish a set of high-level expectations that all FI…"
          }
        },
        {
          "id": "q05",
          "lang": "en",
          "question": "What reputational risk can customer-facing chatbots create?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "Chatbots could provide incorrect information or offensive remarks leading to negative media attention (para 1.9(e))."
          },
          "hit_at_5": true,
          "rank": 4,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 14,
            "score": 0.648,
            "snippet": "e.g., customer-facing systems, such as chatbots, could provide incorrect information or make offensive remarks that lead to negative media attention and reputational damage. 1.9 With Generative AI, such existing risks as…"
          }
        },
        {
          "id": "q06",
          "lang": "en",
          "question": "What security risks does Generative AI introduce, such as prompt injection?",
          "gold": {
            "doc": "guidelines",
            "page": 6,
            "answer": "Adversarial attacks via prompt injection or data poisoning (para 1.10(a))."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 6,
            "score": 0.788,
            "snippet": "AI systems via prompt injection or data poisoning9. (b) Privacy risks, e.g., arising from the leakage of confidential or customer data due to the use of thirdparty Generative AI services, or the inappropriate use of cust…"
          }
        },
        {
          "id": "q07",
          "lang": "en",
          "question": "What extra risks do AI agents with access to tools pose?",
          "gold": {
            "doc": "guidelines",
            "page": 6,
            "answer": "They could autonomously execute unauthorised or erroneous actions; compromised agents could exfiltrate data (para 1.11)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 4,
            "score": 0.727,
            "snippet": "AI providers, and human-factor risks arising from over-reliance on Generative AI. 2.3. The use of even newer technologies, such as AI agents that leverage Generative AI, but with greater autonomy and the ability to acces…"
          }
        },
        {
          "id": "q08",
          "lang": "en",
          "question": "When can a financial institution apply only basic AI governance policies?",
          "gold": {
            "doc": "guidelines",
            "page": 8,
            "answer": "If poor performance or unavailability of the AI is unlikely to have a material adverse impact on the FI, customers or stakeholders (para 2.3)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 5,
            "score": 0.74,
            "snippet": "financial sector is also supported by key initiatives at the national level, such as the Infocomm Media Development Authority’s Model AI Governance Framework9, as well as the various initiatives under the AI Verify Found…"
          }
        },
        {
          "id": "q09",
          "lang": "en",
          "question": "What must basic AI governance policies minimally include?",
          "gold": {
            "doc": "guidelines",
            "page": 9,
            "answer": "Clear accountability for AI oversight, permitted and prohibited uses with human oversight, etc. (para 2.5)."
          },
          "hit_at_5": true,
          "rank": 4,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 12,
            "score": 0.763,
            "snippet": "Media Development Authority’s Model AI Governance Framework, as well as the various initiatives under the AI Verify Foundation to inform their use of AI. 4 Where a point pertains specifically to an AI use case, AI system…"
          }
        },
        {
          "id": "q10",
          "lang": "en",
          "question": "What is the board responsible for regarding AI risk management?",
          "gold": {
            "doc": "guidelines",
            "page": 10,
            "answer": "Approving and reviewing the governance approach, ensuring AI risks are addressed in the risk appetite framework, etc. (para 3.4)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 10,
            "score": 0.825,
            "snippet": "and overseeing robust frameworks, policies and procedures to support AI risk management across the FI. 3.2. The board and senior management should ensure that existing risk management frameworks, policies, and practices …"
          }
        },
        {
          "id": "q11",
          "lang": "en",
          "question": "What should an FI do about residual risk from unidentified or shadow AI use?",
          "gold": {
            "doc": "guidelines",
            "page": 13,
            "answer": "Ensure residual risks stay within risk appetite; identify risks from constraints and put mitigants in place (para 4.4)."
          },
          "hit_at_5": true,
          "rank": 4,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 19,
            "score": 0.718,
            "snippet": "residual risks remain within the risk appetite of the FI. 4.4 For an AI use case, system or model assessed as high risk, an FI should develop and implement contingency plans. These plans should outline fallback options, …"
          }
        },
        {
          "id": "q12",
          "lang": "en",
          "question": "Do FIs need to keep an inventory of AI use cases?",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Yes, an accurate inventory of AI use cases, systems or models across the FI (para 4.5)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 17,
            "score": 0.814,
            "snippet": "maintenance of the inventory, with new, updated or decommissioned AI use cases, systems or models reflected accurately. The FI can enhance existing inventories to include AI use cases, systems or models or establish a de…"
          }
        },
        {
          "id": "q13",
          "lang": "en",
          "question": "Which attributes should an AI inventory record?",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Purpose, approved scope, model type, data used, dependencies, lifecycle status, risk materiality rating, review status, roles, documentation links (para 4.7)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 14,
            "score": 0.769,
            "snippet": "models or establish a dedicated AI inventory. In either case, there should be clear linkages between the AI inventory and other relevant inventories23 in the FI, to facilitate cross-referencing between inventories as par…"
          }
        },
        {
          "id": "q14",
          "lang": "en",
          "question": "Who should oversee the AI inventory to keep it accurate and complete?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "A designated control function with independent oversight (para 4.9)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 15,
            "score": 0.815,
            "snippet": "responsible for establishing policies and procedures relating to the AI inventory. While maintenance, updates and regular reviews of the inventory may be carried out by relevant business or functional units, the designat…"
          }
        },
        {
          "id": "q15",
          "lang": "en",
          "question": "What is the difference between inherent and residual risk materiality?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Inherent is before controls, residual after; residual must meet risk appetite before deployment (para 4.11)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 18,
            "score": 0.677,
            "snippet": "also be regularly reviewed to ensure their continued relevance and appropriateness. 3.10 The risk materiality assessments should consider various risk dimensions relevant to the FI’s context, minimally covering: a. Impac…"
          }
        },
        {
          "id": "q16",
          "lang": "en",
          "question": "What three dimensions must a risk materiality assessment cover?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Impact, complexity and reliance (para 4.12)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 18,
            "score": 0.772,
            "snippet": "also be regularly reviewed to ensure their continued relevance and appropriateness. 3.10 The risk materiality assessments should consider various risk dimensions relevant to the FI’s context, minimally covering: a. Impac…"
          }
        },
        {
          "id": "q17",
          "lang": "en",
          "question": "What does the reliance dimension consider?",
          "gold": {
            "doc": "guidelines",
            "page": 16,
            "answer": "Level of reliance on AI, autonomy granted and degree of human involvement (para 4.12(c))."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 16,
            "score": 0.442,
            "snippet": "that was not as well understood initially may change. For third-party AI, the level of visibility into the nature of the AI technology or the data used should also be considered. (c) Reliance: The level of reliance on AI…"
          }
        },
        {
          "id": "q18",
          "lang": "en",
          "question": "What contingency plans are needed for high-risk AI, including kill switches?",
          "gold": {
            "doc": "guidelines",
            "page": 17,
            "answer": "Fallback options such as alternative systems or manual processes; tested kill-switch activation protocols (para 5.3)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 26,
            "score": 0.775,
            "snippet": "or system, depending on the severity and nature of the issue. For high risk materiality AI, consider implementing \"kill switches\" or override mechanisms to rapidly deactivate the AI system if it exceeds risk tolerances. …"
          }
        },
        {
          "id": "q19",
          "lang": "en",
          "question": "What data management controls are expected for data used in AI?",
          "gold": {
            "doc": "guidelines",
            "page": 17,
            "answer": "Data fit for purpose, representative, high quality and under robust data governance (para 5.4)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 19,
            "score": 0.836,
            "snippet": "data management controls to ensure data used across the AI life cycle is fit for purpose and representative, of high quality, and subject to robust data governance. General data governance and management standards, in ar…"
          }
        },
        {
          "id": "q20",
          "lang": "en",
          "question": "How should human oversight account for automation bias?",
          "gold": {
            "doc": "guidelines",
            "page": 20,
            "answer": "Oversight proportionate to risk, taking into account automation bias and decision fatigue (para 5.9)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 21,
            "score": 0.724,
            "snippet": "across its life cycle. The need for and degree of human oversight should take into account the objective of using AI, and be proportionate to the risk materiality of AI used. It should also take into account automation b…"
          }
        },
        {
          "id": "q21",
          "lang": "en",
          "question": "Does evaluation and testing require clear measurable thresholds before deployment?",
          "gold": {
            "doc": "guidelines",
            "page": 23,
            "answer": "Yes, identify key risks and set clear, measurable thresholds, proportionate to risk materiality (para 5.14)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 23,
            "score": 0.659,
            "snippet": "and establish acceptable performance thresholds for these evaluation measures. Performance thresholds should be clearly defined, documented, and mutually agreed upon by business owners, developers and reviewers. (b) Test…"
          }
        },
        {
          "id": "q22",
          "lang": "en",
          "question": "How detailed must AI development documentation be for auditability?",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Sufficient for an independent reviewer or auditor to understand and potentially replicate it (para 5.17)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 25,
            "score": 0.795,
            "snippet": "Reproducibility and Auditability 5.17. An FI should document the AI development process to enable reproducibility and auditability. The extent and rigour of documentation should be commensurate with the risk materiality …"
          }
        },
        {
          "id": "q23",
          "lang": "en",
          "question": "Who should review an AI use case before deployment?",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Parties not involved in its development (para 5.18)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 23,
            "score": 0.744,
            "snippet": "risk materiality of the applicable AI use case before deployment. An FI should identify key risks arising from the AI use case and set clear, measurable thresholds. Identification of risks and reliability and safety asse…"
          }
        },
        {
          "id": "q24",
          "lang": "en",
          "question": "Why is ongoing monitoring of deployed AI needed?",
          "gold": {
            "doc": "guidelines",
            "page": 27,
            "answer": "Model staleness and performance degradation from data or model drift (para 5.23)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 25,
            "score": 0.843,
            "snippet": "and robust controls for the ongoing monitoring of all deployed AI (including third-party AI used in the FI). Given the uncertainties associated with AI, their dynamic nature and the potential for model staleness and perf…"
          }
        },
        {
          "id": "q25",
          "lang": "en",
          "question": "What should controls for decommissioning AI consider?",
          "gold": {
            "doc": "guidelines",
            "page": 17,
            "answer": "Dependencies, data retention, secure removal from production, stakeholder notifications (para 5.26)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 29,
            "score": 0.749,
            "snippet": "controls for the eventual retirement or decommissioning of AI when they are no longer needed or exceed risk tolerances. These controls should consider dependencies, data retention policies, secure removal from the produc…"
          }
        },
        {
          "id": "q26",
          "lang": "en",
          "question": "Does the guideline mention GPUs and infrastructure adequacy?",
          "gold": {
            "doc": "guidelines",
            "page": 30,
            "answer": "Yes, hardware such as GPUs, network, memory and secure data pipelines must be sufficient (para 6.3)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 30,
            "score": 0.53,
            "snippet": "should ensure that the underlying hardware and software resources, such as graphics processing units, network infrastructure, system memory, secure data pipelines are sufficient to meet performance, scalability, and resi…"
          }
        },
        {
          "id": "q27",
          "lang": "en",
          "question": "When did the consultation on the AI risk management guidelines close?",
          "gold": {
            "doc": "consultation",
            "page": 3,
            "answer": "31 January 2026 (CP para 1.5)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 11,
            "score": 0.694,
            "snippet": "months. 9 Proposed Guidelines on AI Risk Management | 12 6. Proposed Guidelines on AI Risk Management 1 INTRODUCTION 1.1 The Guidelines on Artificial Intelligence (AI) Risk Management (AIRG) set out MAS’ supervisory expe…"
          }
        },
        {
          "id": "q28",
          "lang": "en",
          "question": "What is Project MindForge?",
          "gold": {
            "doc": "consultation",
            "page": 4,
            "answer": "Established to examine Generative AI risks and opportunities; consortium released a GenAI risk framework and works on an AI Risk Management Handbook (CP para 2.5)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "consultation",
            "page": 4,
            "score": 0.667,
            "snippet": "methodologies, a toolkit, and accompanying case studies. 2.5. With the emergence of Generative AI, Project MindForge4 was established to examine the risks and opportunities of Generative AI. The first phase of Project Mi…"
          }
        },
        {
          "id": "q29",
          "lang": "en",
          "question": "What is the Veritas Initiative?",
          "gold": {
            "doc": "consultation",
            "page": 4,
            "answer": "Industry consortium from Nov 2019 helping FIs incorporate FEAT principles; released methodologies and toolkit (CP para 2.4)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "consultation",
            "page": 4,
            "score": 0.48,
            "snippet": "MAS had established key principles to guide FIs in their responsible use of AI. In 2018, MAS co-created the principles of Fairness, Ethics, Accountability and Transparency (FEAT)2 with the financial industry to promote t…"
          }
        },
        {
          "id": "q30",
          "lang": "en",
          "question": "What does FEAT stand for and what are its accuracy and bias principles?",
          "gold": {
            "doc": "feat",
            "page": 6,
            "answer": "Fairness, Ethics, Accountability, Transparency; data and models regularly reviewed and validated for accuracy and to minimise bias (FEAT principles 3-4)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 20,
            "score": 0.44,
            "snippet": "review where pre-defined conditions (e.g., on reliability and accuracy) are met. 35 The consideration of “fair” outcomes may take reference from relevant frameworks and principles, such as MAS’ FEAT principles (https://w…"
          }
        },
        {
          "id": "q31",
          "lang": "en",
          "question": "Can customers appeal or request review of AI-driven decisions under FEAT?",
          "gold": {
            "doc": "feat",
            "page": 6,
            "answer": "Yes, data subjects are provided channels to enquire, appeal and request reviews (FEAT principle 10)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 20,
            "score": 0.659,
            "snippet": "and discriminatory outcomes across the AI life cycle. Controls should be calibrated to the context and nature of the AI use case and its assessed risk materiality. For example, greater attention should be paid to AI used…"
          }
        },
        {
          "id": "q32",
          "lang": "en",
          "question": "Are firms accountable for externally sourced AI models under FEAT?",
          "gold": {
            "doc": "feat",
            "page": 6,
            "answer": "Yes, accountable for both internally developed and externally sourced models (FEAT principle 8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 4,
            "score": 0.697,
            "snippet": "AI providers, and human-factor risks arising from over-reliance on Generative AI. 2.3. The use of even newer technologies, such as AI agents that leverage Generative AI, but with greater autonomy and the ability to acces…"
          }
        }
      ]
    },
    "intfloat/multilingual-e5-small": {
      "model": "intfloat/multilingual-e5-small",
      "dim": 384,
      "index_build_s": 5.8,
      "n_queries": 32,
      "recall_at_5": 0.781,
      "recall_at_1": 0.406,
      "citation_hit_rate": 0.406,
      "mrr_at_5": 0.568,
      "latency_ms": {
        "repeats_per_query": 5,
        "query_embed_plus_search_p50": 7.93,
        "query_embed_plus_search_p95": 122.54,
        "vector_search_only_p50": 0.039,
        "vector_search_only_p95": 0.093
      },
      "per_query": [
        {
          "id": "q01",
          "lang": "en",
          "question": "When do the MAS AI risk management guidelines take effect?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "7 October 2027; Sections 3-4 from 7 Oct 2027, Sections 5-6 by 7 Oct 2028 (para 1.8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 1,
            "score": 0.905,
            "snippet": "Guidelines 7 October 2026 Guidelines on Artificial Intelligence Risk Management 7 OCTOBER 2026 | 2 Contents 1. Introduction 3 2. Application of the Guidelines 8 3. AI Oversight 10 4. Key AI Risk Management Systems, Polic…"
          }
        },
        {
          "id": "q02",
          "lang": "en",
          "question": "By when must FIs meet the AI life cycle controls and capability expectations?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "Sections 5 and 6 by 7 October 2028 (para 1.8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 6,
            "score": 0.894,
            "snippet": "well as the communication, checks and reviews of such guidelines. Expectations relating to AI oversight and key AI risk management systems, policies and procedures would only apply to FIs using AI as an integrated part o…"
          }
        },
        {
          "id": "q03",
          "lang": "en",
          "question": "Are rule-based calculators considered AI under the guidelines?",
          "gold": {
            "doc": "guidelines",
            "page": 4,
            "answer": "No: tools whose outputs are solely based on predefined programming logic or rules are not AI (para 1.3)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 6,
            "score": 0.879,
            "snippet": "manner are set out in paragraph 1.5 and the Annex of the Guidelines. Proposed AIRG | 7 may influence physical or virtual environments, and vary in their levels of autonomy and adaptiveness after deployment.14 Calculators…"
          }
        },
        {
          "id": "q04",
          "lang": "en",
          "question": "How do the guidelines define a model, a system and a use case?",
          "gold": {
            "doc": "guidelines",
            "page": 4,
            "answer": "Model converts assumptions and input data into outputs; system is one or more models plus other components; use case is a specific real-world context (para 1.4)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 4,
            "score": 0.901,
            "snippet": "include estimates, predictions, content, summaries, recommendations, or decisions that may influence physical or virtual environments. Different AI vary in their levels of autonomy and adaptiveness after deployment 6 . C…"
          }
        },
        {
          "id": "q05",
          "lang": "en",
          "question": "What reputational risk can customer-facing chatbots create?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "Chatbots could provide incorrect information or offensive remarks leading to negative media attention (para 1.9(e))."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 14,
            "score": 0.865,
            "snippet": "e.g., customer-facing systems, such as chatbots, could provide incorrect information or make offensive remarks that lead to negative media attention and reputational damage. 1.9 With Generative AI, such existing risks as…"
          }
        },
        {
          "id": "q06",
          "lang": "en",
          "question": "What security risks does Generative AI introduce, such as prompt injection?",
          "gold": {
            "doc": "guidelines",
            "page": 6,
            "answer": "Adversarial attacks via prompt injection or data poisoning (para 1.10(a))."
          },
          "hit_at_5": true,
          "rank": 3,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 14,
            "score": 0.9,
            "snippet": "outputs. The diverse and often opaque data sources used in Generative AI training, coupled with difficulties in evaluating bias of Generative AI outputs, could also result in decisions that lead to unfair customer outcom…"
          }
        },
        {
          "id": "q07",
          "lang": "en",
          "question": "What extra risks do AI agents with access to tools pose?",
          "gold": {
            "doc": "guidelines",
            "page": 6,
            "answer": "They could autonomously execute unauthorised or erroneous actions; compromised agents could exfiltrate data (para 1.11)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 15,
            "score": 0.877,
            "snippet": "mission critical areas, or skill degradation due to over-reliance on Generative AI. 1.10 Similarly, the use of newer technologies such as AI agents, which may be granted greater autonomy and access to tools, could furthe…"
          }
        },
        {
          "id": "q08",
          "lang": "en",
          "question": "When can a financial institution apply only basic AI governance policies?",
          "gold": {
            "doc": "guidelines",
            "page": 8,
            "answer": "If poor performance or unavailability of the AI is unlikely to have a material adverse impact on the FI, customers or stakeholders (para 2.3)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 8,
            "score": 0.881,
            "snippet": "Such controls may be calibrated based on their relevance12 to the AI use cases, systems or models in the FI. Where such aspects are relevant to the AI use case, system or model, the FI may implement them based on the ris…"
          }
        },
        {
          "id": "q09",
          "lang": "en",
          "question": "What must basic AI governance policies minimally include?",
          "gold": {
            "doc": "guidelines",
            "page": 9,
            "answer": "Clear accountability for AI oversight, permitted and prohibited uses with human oversight, etc. (para 2.5)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 9,
            "score": 0.913,
            "snippet": "to assist in designing marketing or internal materials; and (f) Using AI-powered chatbots to assist personnel in locating relevant internal resources, such as policies or procedures. 2.5. Basic AI governance policies and…"
          }
        },
        {
          "id": "q10",
          "lang": "en",
          "question": "What is the board responsible for regarding AI risk management?",
          "gold": {
            "doc": "guidelines",
            "page": 10,
            "answer": "Approving and reviewing the governance approach, ensuring AI risks are addressed in the risk appetite framework, etc. (para 3.4)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 16,
            "score": 0.897,
            "snippet": "by it, is responsible for: a. approving the overall governance approach for AI risk management, including key frameworks, structures, policies and procedures designed to assess and manage the FI’s AI risk on an ongoing b…"
          }
        },
        {
          "id": "q11",
          "lang": "en",
          "question": "What should an FI do about residual risk from unidentified or shadow AI use?",
          "gold": {
            "doc": "guidelines",
            "page": 13,
            "answer": "Ensure residual risks stay within risk appetite; identify risks from constraints and put mitigants in place (para 4.4)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 13,
            "score": 0.898,
            "snippet": "consistent application of the identification process across the FI including the identification of new AI use over time. It should also be the final arbiter on whether a given use constitutes AI use. The designated contr…"
          }
        },
        {
          "id": "q12",
          "lang": "en",
          "question": "Do FIs need to keep an inventory of AI use cases?",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Yes, an accurate inventory of AI use cases, systems or models across the FI (para 4.5)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 14,
            "score": 0.901,
            "snippet": "4.5. An FI should establish and maintain an accurate inventory of AI use cases, systems or models across the FI to the extent possible or practicable, so as to support governance and oversight, as well as risk management…"
          }
        },
        {
          "id": "q13",
          "lang": "en",
          "question": "Which attributes should an AI inventory record?",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Purpose, approved scope, model type, data used, dependencies, lifecycle status, risk materiality rating, review status, roles, documentation links (para 4.7)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 17,
            "score": 0.889,
            "snippet": "maintenance of the inventory, with new, updated or decommissioned AI use cases, systems or models reflected accurately. The FI can enhance existing inventories to include AI use cases, systems or models or establish a de…"
          }
        },
        {
          "id": "q14",
          "lang": "en",
          "question": "Who should oversee the AI inventory to keep it accurate and complete?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "A designated control function with independent oversight (para 4.9)."
          },
          "hit_at_5": true,
          "rank": 4,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 14,
            "score": 0.873,
            "snippet": "4.5. An FI should establish and maintain an accurate inventory of AI use cases, systems or models across the FI to the extent possible or practicable, so as to support governance and oversight, as well as risk management…"
          }
        },
        {
          "id": "q15",
          "lang": "en",
          "question": "What is the difference between inherent and residual risk materiality?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Inherent is before controls, residual after; residual must meet risk appetite before deployment (para 4.11)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 18,
            "score": 0.837,
            "snippet": "that AI usage remains within approved boundaries. 3.9 The assessment should take into account both the inherent risk materiality of an AI use case, system or model before the appropriate risk management controls are appl…"
          }
        },
        {
          "id": "q16",
          "lang": "en",
          "question": "What three dimensions must a risk materiality assessment cover?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Impact, complexity and reliance (para 4.12)."
          },
          "hit_at_5": true,
          "rank": 4,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 18,
            "score": 0.889,
            "snippet": "also be regularly reviewed to ensure their continued relevance and appropriateness. 3.10 The risk materiality assessments should consider various risk dimensions relevant to the FI’s context, minimally covering: a. Impac…"
          }
        },
        {
          "id": "q17",
          "lang": "en",
          "question": "What does the reliance dimension consider?",
          "gold": {
            "doc": "guidelines",
            "page": 16,
            "answer": "Level of reliance on AI, autonomy granted and degree of human involvement (para 4.12(c))."
          },
          "hit_at_5": true,
          "rank": 3,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 18,
            "score": 0.83,
            "snippet": "of the AI technology used, the novelty of its application, or the data it uses. As the understanding of AI technologies evolves across time, this risk dimension could also evolve, e.g., with more research and greater fam…"
          }
        },
        {
          "id": "q18",
          "lang": "en",
          "question": "What contingency plans are needed for high-risk AI, including kill switches?",
          "gold": {
            "doc": "guidelines",
            "page": 17,
            "answer": "Fallback options such as alternative systems or manual processes; tested kill-switch activation protocols (para 5.3)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 19,
            "score": 0.876,
            "snippet": "residual risks remain within the risk appetite of the FI. 4.4 For an AI use case, system or model assessed as high risk, an FI should develop and implement contingency plans. These plans should outline fallback options, …"
          }
        },
        {
          "id": "q19",
          "lang": "en",
          "question": "What data management controls are expected for data used in AI?",
          "gold": {
            "doc": "guidelines",
            "page": 17,
            "answer": "Data fit for purpose, representative, high quality and under robust data governance (para 5.4)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 19,
            "score": 0.888,
            "snippet": "data management controls to ensure data used across the AI life cycle is fit for purpose and representative, of high quality, and subject to robust data governance. General data governance and management standards, in ar…"
          }
        },
        {
          "id": "q20",
          "lang": "en",
          "question": "How should human oversight account for automation bias?",
          "gold": {
            "doc": "guidelines",
            "page": 20,
            "answer": "Oversight proportionate to risk, taking into account automation bias and decision fatigue (para 5.9)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 20,
            "score": 0.881,
            "snippet": "disadvantages for specific groups using appropriate fairness metrics, and documenting the results and any mitigation steps taken. Human Oversight 5.9. An FI should put in place and regularly review controls to ensure app…"
          }
        },
        {
          "id": "q21",
          "lang": "en",
          "question": "Does evaluation and testing require clear measurable thresholds before deployment?",
          "gold": {
            "doc": "guidelines",
            "page": 23,
            "answer": "Yes, identify key risks and set clear, measurable thresholds, proportionate to risk materiality (para 5.14)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 23,
            "score": 0.869,
            "snippet": "risk materiality of the applicable AI use case before deployment. An FI should identify key risks arising from the AI use case and set clear, measurable thresholds. Identification of risks and reliability and safety asse…"
          }
        },
        {
          "id": "q22",
          "lang": "en",
          "question": "How detailed must AI development documentation be for auditability?",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Sufficient for an independent reviewer or auditor to understand and potentially replicate it (para 5.17)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 25,
            "score": 0.909,
            "snippet": "Reproducibility and Auditability 5.17. An FI should document the AI development process to enable reproducibility and auditability. The extent and rigour of documentation should be commensurate with the risk materiality …"
          }
        },
        {
          "id": "q23",
          "lang": "en",
          "question": "Who should review an AI use case before deployment?",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Parties not involved in its development (para 5.18)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 25,
            "score": 0.901,
            "snippet": "selection rationale; (c) training procedures including code versions, environments, and hyperparameters; (d) evaluation measures and performance thresholds, testing approaches, and results; (e) explainability analysis, f…"
          }
        },
        {
          "id": "q24",
          "lang": "en",
          "question": "Why is ongoing monitoring of deployed AI needed?",
          "gold": {
            "doc": "guidelines",
            "page": 27,
            "answer": "Model staleness and performance degradation from data or model drift (para 5.23)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 27,
            "score": 0.901,
            "snippet": "and implement safeguards, such as input validation, throttling and anomaly detection. Post-Deployment Monitoring and Review 5.23. An FI should develop and implement comprehensive and robust controls for the ongoing monit…"
          }
        },
        {
          "id": "q25",
          "lang": "en",
          "question": "What should controls for decommissioning AI consider?",
          "gold": {
            "doc": "guidelines",
            "page": 17,
            "answer": "Dependencies, data retention, secure removal from production, stakeholder notifications (para 5.26)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 26,
            "score": 0.862,
            "snippet": "early actions. Appropriate checks for data drifts (changes in input data distributions), concept drifts (changes in relationships between inputs and outputs), and overall model drifts should be implemented. Where relevan…"
          }
        },
        {
          "id": "q26",
          "lang": "en",
          "question": "Does the guideline mention GPUs and infrastructure adequacy?",
          "gold": {
            "doc": "guidelines",
            "page": 30,
            "answer": "Yes, hardware such as GPUs, network, memory and secure data pipelines must be sufficient (para 6.3)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 30,
            "score": 0.859,
            "snippet": "should ensure that the underlying hardware and software resources, such as graphics processing units, network infrastructure, system memory, secure data pipelines are sufficient to meet performance, scalability, and resi…"
          }
        },
        {
          "id": "q27",
          "lang": "en",
          "question": "When did the consultation on the AI risk management guidelines close?",
          "gold": {
            "doc": "consultation",
            "page": 3,
            "answer": "31 January 2026 (CP para 1.5)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 1,
            "score": 0.888,
            "snippet": "Consultation Paper P017-2025 – November 2025 Consultation Paper on Guidelines on Artificial Intelligence Risk Management Preface | 2 Contents 1. Preface 3 2. MAS’ Supervisory Approach to AI Risk Management 4 3. Applicabi…"
          }
        },
        {
          "id": "q28",
          "lang": "en",
          "question": "What is Project MindForge?",
          "gold": {
            "doc": "consultation",
            "page": 4,
            "answer": "Established to examine Generative AI risks and opportunities; consortium released a GenAI risk framework and works on an AI Risk Management Handbook (CP para 2.5)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "consultation",
            "page": 4,
            "score": 0.82,
            "snippet": "methodologies, a toolkit, and accompanying case studies. 2.5. With the emergence of Generative AI, Project MindForge4 was established to examine the risks and opportunities of Generative AI. The first phase of Project Mi…"
          }
        },
        {
          "id": "q29",
          "lang": "en",
          "question": "What is the Veritas Initiative?",
          "gold": {
            "doc": "consultation",
            "page": 4,
            "answer": "Industry consortium from Nov 2019 helping FIs incorporate FEAT principles; released methodologies and toolkit (CP para 2.4)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "consultation",
            "page": 4,
            "score": 0.832,
            "snippet": "MAS had established key principles to guide FIs in their responsible use of AI. In 2018, MAS co-created the principles of Fairness, Ethics, Accountability and Transparency (FEAT)2 with the financial industry to promote t…"
          }
        },
        {
          "id": "q30",
          "lang": "en",
          "question": "What does FEAT stand for and what are its accuracy and bias principles?",
          "gold": {
            "doc": "feat",
            "page": 6,
            "answer": "Fairness, Ethics, Accountability, Transparency; data and models regularly reviewed and validated for accuracy and to minimise bias (FEAT principles 3-4)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 1,
            "score": 0.883,
            "snippet": "Principles to Promote Fairness, Ethics, Accountability and Transparency (FEAT) in the Use of Artificial Intelligence and Data Analytics in Singapore’s Financial Sector Monetary Authority Of Singapore 1 Principles to Prom…"
          }
        },
        {
          "id": "q31",
          "lang": "en",
          "question": "Can customers appeal or request review of AI-driven decisions under FEAT?",
          "gold": {
            "doc": "feat",
            "page": 6,
            "answer": "Yes, data subjects are provided channels to enquire, appeal and request reviews (FEAT principle 10)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 13,
            "score": 0.872,
            "snippet": "AIDA Firm deploys an AI phone operator to handle customer calls and engage directly with customers. In this case, the AIDA Firm opts to begin the conversation by informing the data subject that they are interacting with …"
          }
        },
        {
          "id": "q32",
          "lang": "en",
          "question": "Are firms accountable for externally sourced AI models under FEAT?",
          "gold": {
            "doc": "feat",
            "page": 6,
            "answer": "Yes, accountable for both internally developed and externally sourced models (FEAT principle 8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 4,
            "score": 0.875,
            "snippet": "MAS had established key principles to guide FIs in their responsible use of AI. In 2018, MAS co-created the principles of Fairness, Ethics, Accountability and Transparency (FEAT)2 with the financial industry to promote t…"
          }
        }
      ]
    }
  },
  "multilingual_probe": {
    "sentence-transformers/all-MiniLM-L6-v2": {
      "model": "sentence-transformers/all-MiniLM-L6-v2",
      "dim": 384,
      "index_build_s": 2.87,
      "n_queries": 10,
      "recall_at_5": 0.0,
      "recall_at_1": 0.0,
      "citation_hit_rate": 0.0,
      "mrr_at_5": 0.0,
      "latency_ms": {
        "repeats_per_query": 5,
        "query_embed_plus_search_p50": 4.62,
        "query_embed_plus_search_p95": 13.71,
        "vector_search_only_p50": 0.038,
        "vector_search_only_p95": 0.043
      },
      "per_query": [
        {
          "id": "m01",
          "lang": "zh",
          "question": "MAS 人工智能风险管理指引什么时候生效？",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "7 October 2027; Sections 3-4 from 7 Oct 2027, Sections 5-6 by 7 Oct 2028 (para 1.8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 12,
            "score": 0.168,
            "snippet": "approach the Firm for a meaningful explanation on the reasons for the increase. The AIDA Firm shares with the customer which customer group he falls into and the corresponding impact on premiums. The AIDA Firm also broad…"
          }
        },
        {
          "id": "m02",
          "lang": "zh",
          "question": "金融机构是否需要维护人工智能用例清单？",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Yes, an accurate inventory of AI use cases, systems or models across the FI (para 4.5)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 14,
            "score": 0.218,
            "snippet": "Hua (Co-Chair), Co-Founder and Advisor, PrimePartners V K Rajah (Special Advisor), Senior Counsel and Member of Essex Court Chambers Duxton (Singapore Group Practice) Teo Swee Lian (Advisor), Non-executive Independent Di…"
          }
        },
        {
          "id": "m03",
          "lang": "zh",
          "question": "人工智能用例上线前应由谁进行审查？",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Parties not involved in its development (para 5.18)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 12,
            "score": 0.15,
            "snippet": "approach the Firm for a meaningful explanation on the reasons for the increase. The AIDA Firm shares with the customer which customer group he falls into and the corresponding impact on premiums. The AIDA Firm also broad…"
          }
        },
        {
          "id": "m04",
          "lang": "zh",
          "question": "为什么需要持续监控已部署的人工智能？",
          "gold": {
            "doc": "guidelines",
            "page": 27,
            "answer": "Model staleness and performance degradation from data or model drift (para 5.23)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 1,
            "score": 0.156,
            "snippet": "Principles to Promote Fairness, Ethics, Accountability and Transparency (FEAT) in the Use of Artificial Intelligence and Data Analytics in Singapore’s Financial Sector Monetary Authority Of Singapore 1 Principles to Prom…"
          }
        },
        {
          "id": "m05",
          "lang": "zh",
          "question": "风险重要性评估必须涵盖哪三个维度？",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Impact, complexity and reliance (para 4.12)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 14,
            "score": 0.232,
            "snippet": "Hua (Co-Chair), Co-Founder and Advisor, PrimePartners V K Rajah (Special Advisor), Senior Counsel and Member of Essex Court Chambers Duxton (Singapore Group Practice) Teo Swee Lian (Advisor), Non-executive Independent Di…"
          }
        },
        {
          "id": "m06",
          "lang": "ms",
          "question": "Bilakah garis panduan pengurusan risiko AI MAS berkuat kuasa?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "7 October 2027; Sections 3-4 from 7 Oct 2027, Sections 5-6 by 7 Oct 2028 (para 1.8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 1,
            "score": 0.365,
            "snippet": "Principles to Promote Fairness, Ethics, Accountability and Transparency (FEAT) in the Use of Artificial Intelligence and Data Analytics in Singapore’s Financial Sector Monetary Authority Of Singapore 1 Principles to Prom…"
          }
        },
        {
          "id": "m07",
          "lang": "ms",
          "question": "Adakah institusi kewangan perlu menyimpan inventori kes penggunaan AI?",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Yes, an accurate inventory of AI use cases, systems or models across the FI (para 4.5)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 1,
            "score": 0.359,
            "snippet": "Principles to Promote Fairness, Ethics, Accountability and Transparency (FEAT) in the Use of Artificial Intelligence and Data Analytics in Singapore’s Financial Sector Monetary Authority Of Singapore 1 Principles to Prom…"
          }
        },
        {
          "id": "m08",
          "lang": "ms",
          "question": "Siapa yang patut menyemak kes penggunaan AI sebelum ia dilancarkan?",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Parties not involved in its development (para 5.18)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 1,
            "score": 0.407,
            "snippet": "Principles to Promote Fairness, Ethics, Accountability and Transparency (FEAT) in the Use of Artificial Intelligence and Data Analytics in Singapore’s Financial Sector Monetary Authority Of Singapore 1 Principles to Prom…"
          }
        },
        {
          "id": "m09",
          "lang": "ms",
          "question": "Mengapakah pemantauan berterusan AI yang telah dilancarkan diperlukan?",
          "gold": {
            "doc": "guidelines",
            "page": 27,
            "answer": "Model staleness and performance degradation from data or model drift (para 5.23)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 13,
            "score": 0.391,
            "snippet": "AIDA Firm deploys an AI phone operator to handle customer calls and engage directly with customers. In this case, the AIDA Firm opts to begin the conversation by informing the data subject that they are interacting with …"
          }
        },
        {
          "id": "m10",
          "lang": "ms",
          "question": "Apakah tiga dimensi yang mesti diliputi oleh penilaian kepentingan risiko?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Impact, complexity and reliance (para 4.12)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 1,
            "score": 0.297,
            "snippet": "Principles to Promote Fairness, Ethics, Accountability and Transparency (FEAT) in the Use of Artificial Intelligence and Data Analytics in Singapore’s Financial Sector Monetary Authority Of Singapore 1 Principles to Prom…"
          }
        }
      ]
    },
    "BAAI/bge-small-en-v1.5": {
      "model": "BAAI/bge-small-en-v1.5",
      "dim": 384,
      "index_build_s": 5.91,
      "n_queries": 10,
      "recall_at_5": 0.0,
      "recall_at_1": 0.0,
      "citation_hit_rate": 0.0,
      "mrr_at_5": 0.0,
      "latency_ms": {
        "repeats_per_query": 5,
        "query_embed_plus_search_p50": 8.2,
        "query_embed_plus_search_p95": 82.42,
        "vector_search_only_p50": 0.039,
        "vector_search_only_p95": 0.067
      },
      "per_query": [
        {
          "id": "m01",
          "lang": "zh",
          "question": "MAS 人工智能风险管理指引什么时候生效？",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "7 October 2027; Sections 3-4 from 7 Oct 2027, Sections 5-6 by 7 Oct 2028 (para 1.8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 3,
            "score": 0.648,
            "snippet": "their identity), or (b) their identity along with their whole submission, to be kept confidential, please expressly state so in the submission to MAS. MAS will only publish nonanonymous submissions. In addition, MAS rese…"
          }
        },
        {
          "id": "m02",
          "lang": "zh",
          "question": "金融机构是否需要维护人工智能用例清单？",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Yes, an accurate inventory of AI use cases, systems or models across the FI (para 4.5)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 8,
            "score": 0.612,
            "snippet": "notes for internal reference; 12 Certain AI life cycle controls, such as those relating to transparency, explainability, and fairness may be more relevant for AI used in areas such as credit scoring, insurance underwriti…"
          }
        },
        {
          "id": "m03",
          "lang": "zh",
          "question": "人工智能用例上线前应由谁进行审查？",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Parties not involved in its development (para 5.18)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 12,
            "score": 0.637,
            "snippet": "and allocating adequate resources (such as human, technological, financial resources) for effective AI risk management, including appropriate training and capacity building. 3.6. Local senior management remains accountab…"
          }
        },
        {
          "id": "m04",
          "lang": "zh",
          "question": "为什么需要持续监控已部署的人工智能？",
          "gold": {
            "doc": "guidelines",
            "page": 27,
            "answer": "Model staleness and performance degradation from data or model drift (para 5.23)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 3,
            "score": 0.629,
            "snippet": "their identity), or (b) their identity along with their whole submission, to be kept confidential, please expressly state so in the submission to MAS. MAS will only publish nonanonymous submissions. In addition, MAS rese…"
          }
        },
        {
          "id": "m05",
          "lang": "zh",
          "question": "风险重要性评估必须涵盖哪三个维度？",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Impact, complexity and reliance (para 4.12)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 10,
            "score": 0.623,
            "snippet": "management that is responsible for the oversight of the FI in Singapore, such as a regional or global committee which exercises oversight of the risks from AI use relevant to the FI in Singapore. For an FI which is not p…"
          }
        },
        {
          "id": "m06",
          "lang": "ms",
          "question": "Bilakah garis panduan pengurusan risiko AI MAS berkuat kuasa?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "7 October 2027; Sections 3-4 from 7 Oct 2027, Sections 5-6 by 7 Oct 2028 (para 1.8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 14,
            "score": 0.518,
            "snippet": "the following contributors, for providing valuable inputs and feedback to the FEAT Principles. (In alphabetical order) Amazon Web Services American Express Int’l Inc Asia Cloud Computing Association Aviva Ltd Barclays Ba…"
          }
        },
        {
          "id": "m07",
          "lang": "ms",
          "question": "Adakah institusi kewangan perlu menyimpan inventori kes penggunaan AI?",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Yes, an accurate inventory of AI use cases, systems or models across the FI (para 4.5)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 1,
            "score": 0.549,
            "snippet": "Principles to Promote Fairness, Ethics, Accountability and Transparency (FEAT) in the Use of Artificial Intelligence and Data Analytics in Singapore’s Financial Sector Monetary Authority Of Singapore 1 Principles to Prom…"
          }
        },
        {
          "id": "m08",
          "lang": "ms",
          "question": "Siapa yang patut menyemak kes penggunaan AI sebelum ia dilancarkan?",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Parties not involved in its development (para 5.18)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 1,
            "score": 0.533,
            "snippet": "Consultation Paper P017-2025 – November 2025 Consultation Paper on Guidelines on Artificial Intelligence Risk Management Preface | 2 Contents 1. Preface 3 2. MAS’ Supervisory Approach to AI Risk Management 4 3. Applicabi…"
          }
        },
        {
          "id": "m09",
          "lang": "ms",
          "question": "Mengapakah pemantauan berterusan AI yang telah dilancarkan diperlukan?",
          "gold": {
            "doc": "guidelines",
            "page": 27,
            "answer": "Model staleness and performance degradation from data or model drift (para 5.23)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 1,
            "score": 0.561,
            "snippet": "Consultation Paper P017-2025 – November 2025 Consultation Paper on Guidelines on Artificial Intelligence Risk Management Preface | 2 Contents 1. Preface 3 2. MAS’ Supervisory Approach to AI Risk Management 4 3. Applicabi…"
          }
        },
        {
          "id": "m10",
          "lang": "ms",
          "question": "Apakah tiga dimensi yang mesti diliputi oleh penilaian kepentingan risiko?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Impact, complexity and reliance (para 4.12)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "feat",
            "page": 1,
            "score": 0.509,
            "snippet": "Principles to Promote Fairness, Ethics, Accountability and Transparency (FEAT) in the Use of Artificial Intelligence and Data Analytics in Singapore’s Financial Sector Monetary Authority Of Singapore 1 Principles to Prom…"
          }
        }
      ]
    },
    "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2": {
      "model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
      "dim": 384,
      "index_build_s": 4.15,
      "n_queries": 10,
      "recall_at_5": 0.2,
      "recall_at_1": 0.0,
      "citation_hit_rate": 0.1,
      "mrr_at_5": 0.07,
      "latency_ms": {
        "repeats_per_query": 5,
        "query_embed_plus_search_p50": 7.4,
        "query_embed_plus_search_p95": 89.37,
        "vector_search_only_p50": 0.044,
        "vector_search_only_p95": 0.069
      },
      "per_query": [
        {
          "id": "m01",
          "lang": "zh",
          "question": "MAS 人工智能风险管理指引什么时候生效？",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "7 October 2027; Sections 3-4 from 7 Oct 2027, Sections 5-6 by 7 Oct 2028 (para 1.8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 3,
            "score": 0.794,
            "snippet": "and set out MAS’ supervisory expectations relating to AI risk management in the financial sector. The Guidelines focus on oversight of AI risk management in FIs, key AI risk management systems, policies and procedures, k…"
          }
        },
        {
          "id": "m02",
          "lang": "zh",
          "question": "金融机构是否需要维护人工智能用例清单？",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Yes, an accurate inventory of AI use cases, systems or models across the FI (para 4.5)."
          },
          "hit_at_5": true,
          "rank": 5,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 3,
            "score": 0.726,
            "snippet": "time to time. For instance, existing principles on Fairness, Ethics, Accountability and Transparency (FEAT)2 continue to apply for guiding the use of AI in the financial sector. FIs should also take into account applicab…"
          }
        },
        {
          "id": "m03",
          "lang": "zh",
          "question": "人工智能用例上线前应由谁进行审查？",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Parties not involved in its development (para 5.18)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 25,
            "score": 0.717,
            "snippet": "suitability, robustness, and performance of AI for its intended purpose within defined boundaries. Roles and responsibilities relating to such reviews should be clearly assigned, and the scope and independence of these p…"
          }
        },
        {
          "id": "m04",
          "lang": "zh",
          "question": "为什么需要持续监控已部署的人工智能？",
          "gold": {
            "doc": "guidelines",
            "page": 27,
            "answer": "Model staleness and performance degradation from data or model drift (para 5.23)."
          },
          "hit_at_5": true,
          "rank": 2,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 25,
            "score": 0.8,
            "snippet": "and robust controls for the ongoing monitoring of all deployed AI (including third-party AI used in the FI). Given the uncertainties associated with AI, their dynamic nature and the potential for model staleness and perf…"
          }
        },
        {
          "id": "m05",
          "lang": "zh",
          "question": "风险重要性评估必须涵盖哪三个维度？",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Impact, complexity and reliance (para 4.12)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 18,
            "score": 0.742,
            "snippet": "also be regularly reviewed to ensure their continued relevance and appropriateness. 3.10 The risk materiality assessments should consider various risk dimensions relevant to the FI’s context, minimally covering: a. Impac…"
          }
        },
        {
          "id": "m06",
          "lang": "ms",
          "question": "Bilakah garis panduan pengurusan risiko AI MAS berkuat kuasa?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "7 October 2027; Sections 3-4 from 7 Oct 2027, Sections 5-6 by 7 Oct 2028 (para 1.8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 8,
            "score": 0.845,
            "snippet": "management standards, and controls effectively to such usage of AI. Hence MAS expects FIs to establish clear definitions, criteria and processes, supported by robust systems, to facilitate this identification process and…"
          }
        },
        {
          "id": "m07",
          "lang": "ms",
          "question": "Adakah institusi kewangan perlu menyimpan inventori kes penggunaan AI?",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Yes, an accurate inventory of AI use cases, systems or models across the FI (para 4.5)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 13,
            "score": 0.717,
            "snippet": "FIs using AI as an integrated part of their business processes should minimally establish frameworks, policies and procedures to oversee their use of AI; apply clear identification and robust risk materiality assessments…"
          }
        },
        {
          "id": "m08",
          "lang": "ms",
          "question": "Siapa yang patut menyemak kes penggunaan AI sebelum ia dilancarkan?",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Parties not involved in its development (para 5.18)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 9,
            "score": 0.692,
            "snippet": "of AI outputs prior to use is required); (c) maintenance of an approved list of AI tools, and a defined process for requesting approval of new AI tools; (d) staff education on the FI’s AI-related policies; (e) regular co…"
          }
        },
        {
          "id": "m09",
          "lang": "ms",
          "question": "Mengapakah pemantauan berterusan AI yang telah dilancarkan diperlukan?",
          "gold": {
            "doc": "guidelines",
            "page": 27,
            "answer": "Model staleness and performance degradation from data or model drift (para 5.23)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 25,
            "score": 0.734,
            "snippet": "and robust controls for the ongoing monitoring of all deployed AI (including third-party AI used in the FI). Given the uncertainties associated with AI, their dynamic nature and the potential for model staleness and perf…"
          }
        },
        {
          "id": "m10",
          "lang": "ms",
          "question": "Apakah tiga dimensi yang mesti diliputi oleh penilaian kepentingan risiko?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Impact, complexity and reliance (para 4.12)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 18,
            "score": 0.681,
            "snippet": "also be regularly reviewed to ensure their continued relevance and appropriateness. 3.10 The risk materiality assessments should consider various risk dimensions relevant to the FI’s context, minimally covering: a. Impac…"
          }
        }
      ]
    },
    "intfloat/multilingual-e5-small": {
      "model": "intfloat/multilingual-e5-small",
      "dim": 384,
      "index_build_s": 6.48,
      "n_queries": 10,
      "recall_at_5": 0.4,
      "recall_at_1": 0.1,
      "citation_hit_rate": 0.2,
      "mrr_at_5": 0.16,
      "latency_ms": {
        "repeats_per_query": 5,
        "query_embed_plus_search_p50": 9.83,
        "query_embed_plus_search_p95": 101.92,
        "vector_search_only_p50": 0.042,
        "vector_search_only_p95": 0.084
      },
      "per_query": [
        {
          "id": "m01",
          "lang": "zh",
          "question": "MAS 人工智能风险管理指引什么时候生效？",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "7 October 2027; Sections 3-4 from 7 Oct 2027, Sections 5-6 by 7 Oct 2028 (para 1.8)."
          },
          "hit_at_5": true,
          "rank": 5,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 1,
            "score": 0.869,
            "snippet": "Guidelines 7 October 2026 Guidelines on Artificial Intelligence Risk Management 7 OCTOBER 2026 | 2 Contents 1. Introduction 3 2. Application of the Guidelines 8 3. AI Oversight 10 4. Key AI Risk Management Systems, Polic…"
          }
        },
        {
          "id": "m02",
          "lang": "zh",
          "question": "金融机构是否需要维护人工智能用例清单？",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Yes, an accurate inventory of AI use cases, systems or models across the FI (para 4.5)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 5,
            "score": 0.837,
            "snippet": "financial sector is also supported by key initiatives at the national level, such as the Infocomm Media Development Authority’s Model AI Governance Framework9, as well as the various initiatives under the AI Verify Found…"
          }
        },
        {
          "id": "m03",
          "lang": "zh",
          "question": "人工智能用例上线前应由谁进行审查？",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Parties not involved in its development (para 5.18)."
          },
          "hit_at_5": true,
          "rank": 5,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 25,
            "score": 0.81,
            "snippet": "suitability, robustness, and performance of AI for its intended purpose within defined boundaries. Roles and responsibilities relating to such reviews should be clearly assigned, and the scope and independence of these p…"
          }
        },
        {
          "id": "m04",
          "lang": "zh",
          "question": "为什么需要持续监控已部署的人工智能？",
          "gold": {
            "doc": "guidelines",
            "page": 27,
            "answer": "Model staleness and performance degradation from data or model drift (para 5.23)."
          },
          "hit_at_5": true,
          "rank": 1,
          "citation_hit": true,
          "top1": {
            "doc": "guidelines",
            "page": 27,
            "score": 0.811,
            "snippet": "and implement safeguards, such as input validation, throttling and anomaly detection. Post-Deployment Monitoring and Review 5.23. An FI should develop and implement comprehensive and robust controls for the ongoing monit…"
          }
        },
        {
          "id": "m05",
          "lang": "zh",
          "question": "风险重要性评估必须涵盖哪三个维度？",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Impact, complexity and reliance (para 4.12)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 18,
            "score": 0.825,
            "snippet": "also be regularly reviewed to ensure their continued relevance and appropriateness. 3.10 The risk materiality assessments should consider various risk dimensions relevant to the FI’s context, minimally covering: a. Impac…"
          }
        },
        {
          "id": "m06",
          "lang": "ms",
          "question": "Bilakah garis panduan pengurusan risiko AI MAS berkuat kuasa?",
          "gold": {
            "doc": "guidelines",
            "page": 5,
            "answer": "7 October 2027; Sections 3-4 from 7 Oct 2027, Sections 5-6 by 7 Oct 2028 (para 1.8)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "guidelines",
            "page": 1,
            "score": 0.871,
            "snippet": "Guidelines 7 October 2026 Guidelines on Artificial Intelligence Risk Management 7 OCTOBER 2026 | 2 Contents 1. Introduction 3 2. Application of the Guidelines 8 3. AI Oversight 10 4. Key AI Risk Management Systems, Polic…"
          }
        },
        {
          "id": "m07",
          "lang": "ms",
          "question": "Adakah institusi kewangan perlu menyimpan inventori kes penggunaan AI?",
          "gold": {
            "doc": "guidelines",
            "page": 14,
            "answer": "Yes, an accurate inventory of AI use cases, systems or models across the FI (para 4.5)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 17,
            "score": 0.861,
            "snippet": "in determining whether AI is being used. The designated control function should also ensure that clear documentation of the identification process and outcomes is maintained, and that identification systems and processes…"
          }
        },
        {
          "id": "m08",
          "lang": "ms",
          "question": "Siapa yang patut menyemak kes penggunaan AI sebelum ia dilancarkan?",
          "gold": {
            "doc": "guidelines",
            "page": 25,
            "answer": "Parties not involved in its development (para 5.18)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 9,
            "score": 0.834,
            "snippet": "relevance to the AI model, system or use case, and apply in a proportionate manner include data management, fairness, transparency and explainability, human oversight, management of third-party AI risks, selection of AI,…"
          }
        },
        {
          "id": "m09",
          "lang": "ms",
          "question": "Mengapakah pemantauan berterusan AI yang telah dilancarkan diperlukan?",
          "gold": {
            "doc": "guidelines",
            "page": 27,
            "answer": "Model staleness and performance degradation from data or model drift (para 5.23)."
          },
          "hit_at_5": true,
          "rank": 5,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 10,
            "score": 0.832,
            "snippet": "systems, to facilitate the consistent identification of AI across all relevant business and functional areas. 8 Question 6 MAS seeks comments on the proposal for FIs to establish and maintain an accurate and up-to-date i…"
          }
        },
        {
          "id": "m10",
          "lang": "ms",
          "question": "Apakah tiga dimensi yang mesti diliputi oleh penilaian kepentingan risiko?",
          "gold": {
            "doc": "guidelines",
            "page": 15,
            "answer": "Impact, complexity and reliance (para 4.12)."
          },
          "hit_at_5": false,
          "rank": null,
          "citation_hit": false,
          "top1": {
            "doc": "consultation",
            "page": 18,
            "score": 0.852,
            "snippet": "also be regularly reviewed to ensure their continued relevance and appropriateness. 3.10 The risk materiality assessments should consider various risk dimensions relevant to the FI’s context, minimally covering: a. Impac…"
          }
        }
      ]
    }
  }
}