0},e}(),Zn=typeof WeakMap!="undefined"?new WeakMap:new Bn,eo=function(){function e(t){if(!(this instanceof e))throw new TypeError("Cannot call a class as a function.");if(!arguments.length)throw new TypeError("1 argument required, but only 0 present.");var r=Ea.getInstance(),n=new Ra(t,r,this);Zn.set(this,n)}return e}();["observe","unobserve","disconnect"].forEach(function(e){eo.prototype[e]=function(){var t;return(t=Zn.get(this))[e].apply(t,arguments)}});var ka=function(){return typeof or.ResizeObserver!="undefined"?or.ResizeObserver:eo}(),to=ka;var ro=new E,Ha=I(()=>H(new to(e=>{for(let t of e)ro.next(t)}))).pipe(x(e=>L(Oe,H(e)).pipe(C(()=>e.disconnect()))),J(1));function he(e){return{width:e.offsetWidth,height:e.offsetHeight}}function ge(e){return Ha.pipe(S(t=>t.observe(e)),x(t=>ro.pipe(_(({target:r})=>r===e),C(()=>t.unobserve(e)),l(()=>he(e)))),N(he(e)))}function bt(e){return{width:e.scrollWidth,height:e.scrollHeight}}function sr(e){let t=e.parentElement;for(;t&&(e.scrollWidth<=t.scrollWidth&&e.scrollHeight<=t.scrollHeight);)t=(e=t).parentElement;return t?e:void 0}var no=new E,Pa=I(()=>H(new IntersectionObserver(e=>{for(let t of e)no.next(t)},{threshold:0}))).pipe(x(e=>L(Oe,H(e)).pipe(C(()=>e.disconnect()))),J(1));function cr(e){return Pa.pipe(S(t=>t.observe(e)),x(t=>no.pipe(_(({target:r})=>r===e),C(()=>t.unobserve(e)),l(({isIntersecting:r})=>r))))}function oo(e,t=16){return dt(e).pipe(l(({y:r})=>{let n=he(e),o=bt(e);return r>=o.height-n.height-t}),G())}var fr={drawer:V("[data-md-toggle=drawer]"),search:V("[data-md-toggle=search]")};function io(e){return fr[e].checked}function qe(e,t){fr[e].checked!==t&&fr[e].click()}function Ue(e){let t=fr[e];return b(t,"change").pipe(l(()=>t.checked),N(t.checked))}function $a(e,t){switch(e.constructor){case HTMLInputElement:return e.type==="radio"?/^Arrow/.test(t):!0;case HTMLSelectElement:case HTMLTextAreaElement:return!0;default:return e.isContentEditable}}function Ia(){return L(b(window,"compositionstart").pipe(l(()=>!0)),b(window,"compositionend").pipe(l(()=>!1))).pipe(N(!1))}function ao(){let e=b(window,"keydown").pipe(_(t=>!(t.metaKey||t.ctrlKey)),l(t=>({mode:io("search")?"search":"global",type:t.key,claim(){t.preventDefault(),t.stopPropagation()}})),_(({mode:t,type:r})=>{if(t==="global"){let n=_e();if(typeof n!="undefined")return!$a(n,r)}return!0}),ue());return Ia().pipe(x(t=>t?R:e))}function Me(){return new URL(location.href)}function ot(e){location.href=e.href}function so(){return new E}function co(e,t){if(typeof t=="string"||typeof t=="number")e.innerHTML+=t.toString();else if(t instanceof Node)e.appendChild(t);else if(Array.isArray(t))for(let r of t)co(e,r)}function M(e,t,...r){let n=document.createElement(e);if(t)for(let o of Object.keys(t))typeof t[o]!="undefined"&&(typeof t[o]!="boolean"?n.setAttribute(o,t[o]):n.setAttribute(o,""));for(let o of r)co(n,o);return n}function ur(e){if(e>999){let t=+((e-950)%1e3>99);return`${((e+1e-6)/1e3).toFixed(t)}k`}else return e.toString()}function fo(){return location.hash.substring(1)}function uo(e){let t=M("a",{href:e});t.addEventListener("click",r=>r.stopPropagation()),t.click()}function Fa(){return b(window,"hashchange").pipe(l(fo),N(fo()),_(e=>e.length>0),J(1))}function po(){return Fa().pipe(l(e=>ce(`[id="${e}"]`)),_(e=>typeof e!="undefined"))}function Nr(e){let t=matchMedia(e);return er(r=>t.addListener(()=>r(t.matches))).pipe(N(t.matches))}function lo(){let e=matchMedia("print");return L(b(window,"beforeprint").pipe(l(()=>!0)),b(window,"afterprint").pipe(l(()=>!1))).pipe(N(e.matches))}function qr(e,t){return e.pipe(x(r=>r?t():R))}function pr(e,t={credentials:"same-origin"}){return pe(fetch(`${e}`,t)).pipe(fe(()=>R),x(r=>r.status!==200?Ot(()=>new Error(r.statusText)):H(r)))}function We(e,t){return pr(e,t).pipe(x(r=>r.json()),J(1))}function mo(e,t){let r=new DOMParser;return pr(e,t).pipe(x(n=>n.text()),l(n=>r.parseFromString(n,"text/xml")),J(1))}function lr(e){let t=M("script",{src:e});return I(()=>(document.head.appendChild(t),L(b(t,"load"),b(t,"error").pipe(x(()=>Ot(()=>new ReferenceError(`Invalid script: ${e}`))))).pipe(l(()=>{}),C(()=>document.head.removeChild(t)),Te(1))))}function ho(){return{x:Math.max(0,scrollX),y:Math.max(0,scrollY)}}function bo(){return L(b(window,"scroll",{passive:!0}),b(window,"resize",{passive:!0})).pipe(l(ho),N(ho()))}function vo(){return{width:innerWidth,height:innerHeight}}function go(){return b(window,"resize",{passive:!0}).pipe(l(vo),N(vo()))}function yo(){return Q([bo(),go()]).pipe(l(([e,t])=>({offset:e,size:t})),J(1))}function mr(e,{viewport$:t,header$:r}){let n=t.pipe(X("size")),o=Q([n,r]).pipe(l(()=>Je(e)));return Q([r,t,o]).pipe(l(([{height:i},{offset:s,size:a},{x:f,y:c}])=>({offset:{x:s.x-f,y:s.y-c+i},size:a})))}(()=>{function e(n,o){parent.postMessage(n,o||"*")}function t(...n){return n.reduce((o,i)=>o.then(()=>new Promise(s=>{let a=document.createElement("script");a.src=i,a.onload=s,document.body.appendChild(a)})),Promise.resolve())}var r=class extends EventTarget{constructor(n){super(),this.url=n,this.m=i=>{i.source===this.w&&(this.dispatchEvent(new MessageEvent("message",{data:i.data})),this.onmessage&&this.onmessage(i))},this.e=(i,s,a,f,c)=>{if(s===`${this.url}`){let u=new ErrorEvent("error",{message:i,filename:s,lineno:a,colno:f,error:c});this.dispatchEvent(u),this.onerror&&this.onerror(u)}};let o=document.createElement("iframe");o.hidden=!0,document.body.appendChild(this.iframe=o),this.w.document.open(),this.w.document.write(`
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Features
+Design decisions
+
+Easy : Very easy to set up and get started with.Intuitive : Designed to be intuitive and easy to use.Pythonic SDK : Pythonic SDK for building your own monitoring infrastructure.Robust : Get production-ready MLOps system.Kubernetes : Get production-ready code. With automatic interactive documentation.
+Descriptive Statistics
+Whitebox provides a nice list of descriptive statistics of input dataset, making the overview of data easy.
+Models Metrics
+Classification Models
+Whitebox includes comprehensive metrics tracking for classification models. This allows users to easily evaluate the performance of their classification models and identify areas for improvement. Additionally, users can set custom thresholds for each metric to receive alerts when performance deviates from expected results.
+Regression Models
+Whitebox includes comprehensive metrics tracking for regression models. This allows users to easily evaluate the performance of their regression models and identify areas for improvement. Additionally, users can set custom thresholds for each metric to receive alerts when performance deviates from expected results.
+Data / Concept Drift Monitoring
+Whitebox includes monitoring for data and concept drift . This feature tracks changes in the distribution of the data used to train models and alerts users when significant changes occur. Additionally, it detects changes in the performance of deployed models and alerts users when significant drift is detected. This allows users to identify and address data and model drift early, reducing the risk of poor model performance.
+Explainable AI
+Whitebox includes model explaination also. The explainability performed through the explainability report which allows user to know anytime which feature had the most impact on model's prediction.
+Alerts
+Whitebox includes an alerting system that allows users to set custom thresholds for metrics and receive notifications when performance deviates from expected results. These alerts can be delivered via email, SMS, or webhook, and can be customized to fit the needs of any organization. This allows users to quickly respond to changes in model performance and take action to improve results.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Whitebox
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Whitebox
+
+
+
+
+
+ Whitebox is an open source E2E ML monitoring platform with edge capabilities that plays nicely with kubernetes
+
+
+
+Documentation : https://whitebox-ai.github.io/whitebox/
+Source Code : https://github.com/whitebox-ai/whitebox
+Roadmap : https://github.com/whitebox-ai/whitebox/milestones
+Issue tracking https://github.com/orgs/whitebox-ai/projects/1/views/3
+Discord : https://discord.gg/bkAcsx4V
+Whitebox is a dynamic open source E2E ML monitoring platform with edge capabilities that plays nicely with kubernetes.
+The current key features are:
+
+Descriptive statistics Classification models evaluation metrics Data / Concept drift monitoring Explainable AI Alerts
+Design guidelines:
+
+Easy : Very easy to set up and get started with.Intuitive : Designed to be intuitive and easy to use.Pythonic SDK : Pythonic SDK for building your own monitoring infrastructure.Robust : Get production-ready MLOps system.Kubernetes : Get production-ready code. With automatic interactive documentation.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Glossary / Metric definitions - Whitebox
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Glossary / Metric definitions
+Descriptive statistics
+Missing values
+Missing values metric calculates the summary of the number of missing values per feature. Missing values include NaN in numeric arrays, NaN or None in object arrays and NaT in datetimelike.
+Non-Missing values
+Non-Missing values metric calculates the summary of the number of non-missing values per feature. Non-Missing values are all values beside NaN for numeric arrays, NaN or None for object arrays and NaT for datetimelike.
+Mean or Average value
+Returns the average value per feature excluding NaN and null values.
+Minimum value
+Returns the minimum value per feature.
+Maximum value
+Returns the maximum value per feature.
+Summary
+Returns the summary of the values per feature. Excludes NaN and null values during calculations.
+Standard Deviation
+Returns the sample standard deviation per feature normalized by N-1 excluding NaN and null values during calculations. Formula:
+
+
+σ = \sqrt{Σ(x_i-μ)^2 \over Ν-1}
+
+
+
Variance
+Returns the unbiased variance per feature normalized by N-1 excluding NaN and null values during calculations. Formula:
+
+
+σ^2 = {Σ(x_i-μ)^2 \over Ν-1}
+
+
+
Classification evaluation metrics
+Confusion Matrix
+Returns the number of TP, TN, FP and FN. In case of a multi-class classification returns the number of TP, TN, FP and FN per class.
+A typical example for binary classification could be seen below in which:
+
+20 observations were correctly classified as positive.
+10 observations were incorrectly classified as negative while they were actually positive.
+5 observations were incorrectly classified as positive while they were actually negative.
+75 observations were correctly classified as negative.
+
+
+
+
+Predicted Positive
+Predicted Negative
+
+
+
+
+Actual Positive
+20 (TP)
+10 (FN)
+
+
+Actual Negative
+5 (FP)
+75 (TN)
+
+
+
+A typical example for multi-class classification could be seen below in which:
+
+15 observations were correctly classified as Class A.
+5 observations were incorrectly classified as Class B while they were actually Class A.
+2 observations were incorrectly classified as Class C while they were actually Class A.
+4 observations were incorrectly classified as Class A while they were actually Class B.
+20 observations were correctly classified as Class B.
+3 observations were incorrectly classified as Class C while they were actually Class B.
+2 observations were incorrectly classified as Class A while they were actually Class C.
+8 observations were incorrectly classified as Class B while they were actually Class C.
+25 observations were correctly classified as Class C.
+
+
+
+
+Predicted Class A
+Predicted Class B
+Predicted Class C
+
+
+
+
+Actual Class A
+15 (TP_A)
+5
+2
+
+
+Actual Class B
+4
+20 (TP_B)
+3
+
+
+Actual Class C
+2
+8
+25 (TP_C)
+
+
+
+Accuracy
+Returns the accuracy classification score. In multi-class classification, this function computes subset accuracy: the set of labels predicted for a sample must exactly match the corresponding set of labels in y_true. Formula:
+
+
+accuracy = {(TP + TN) \over (TP + TN + FP + FN)}
+
+
+
Precision
+Returns the precision classification score. In multi-class classification, returns the below 3 scores:
+
+micro: Calculate metrics globally by counting the total true positives, false negatives and false positives.macro: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account.weighted: Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters ‘macro’ to account for label imbalance; it can result in an F-score that is not between precision and recall.
+Formula:
+
+
+precision = {TP \over (TP + FP)}
+
+
+
Recall
+Returns the recall classification score. In multi-class classification, returns the below 3 scores:
+
+micro: Calculate metrics globally by counting the total true positives, false negatives and false positives.macro: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account.weighted: Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters ‘macro’ to account for label imbalance; it can result in an F-score that is not between precision and recall.
+Formula:
+
+
+recall = {TP \over (TP + FN)}
+
+
+
F1 score
+Returns the f1 classification score. In multi-class classification, returns the below 3 scores:
+
+micro: Calculate metrics globally by counting the total true positives, false negatives and false positives.macro: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account.weighted: Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters ‘macro’ to account for label imbalance; it can result in an F-score that is not between precision and recall.
+Formula:
+
+
+F1 = 2 * {(precision * recall) \over (precision + recall)}
+
+
+
Regression evaluation metrics
+R-Squared
+In a regression model, R-Squared (also known as R2 or the coefficient of determination) is a statistical metric that quantifies how much of the variance in the dependent variable can be accounted for by the independent variable. R-squared, thus, displays how well the data match the regression model (the goodness of fit).
+Formula:
+
+
+R^2 = {SS_{regression} \over SS_{total}}
+
+
+
where:
+
+SS_{regression} SS_{total}
+Mean squared error
+The degree of inaccuracy in statistical models is gauged by the mean squared error, or MSE. Between the observed and projected values, it evaluates the average squared difference. The MSE is equal to 0 when a model is error-free. As model error increases, its value increases.
+Formula:
+
+
+MSE = {Σ(y_{i}-y'_{i})^2 \over n}
+
+
+
where:
+
+y_{i} i_{th} y'_{i} n
+Mean absolute error
+The mean absolute error of a model with respect to a test set is the mean of the absolute values of the individual prediction errors on over all instances in the test set. Each prediction error is the difference between the true value and the predicted value for the instance.
+Formula:
+
+
+MAE = {{{Σ^2}_{i=1}}|y_{i}-x_{i}| \over n}
+
+
+
where:
+
+y_{i} x_{i} n
+Statistical tests and techniques
+Kolmogorov-Smirnov Two Sample test
+When there are two datasets then K-S two sample test can be used to test the agreement between their distributions. The null hypothesis states that there is no difference between the two distributions. Formula:
+
+
+D = Max|{F_a(X)-F_b(X)}|
+
+
+
where:
+
+a b F_n(X)
+Chi-squared test
+A chi-square test is a statistical test used to compare 2 datasets. The purpose of this test is to determine if a difference between data of 2 datasets is due to chance, or if it is due to a relationship between the variables you are studying. Formula:
+
+
+x^2 = Σ{(O_i - E_i)^2 \over E_i}
+
+
+
where:
+
+x^2 O_i E_i
+Z-score for independent proportions
+The purpose of the z-test for independent proportions is to compare two independent datasets. Formula:
+
+
+Z = {p_1 - p_2 \over \sqrt{p' q' ({1\over n_1} + {1\over n_2})}}
+
+
+
where:
+
+Z p_1 , p_2 p' q' (1-p') n_1 , n_2
+Wasserstein distance
+The Wasserstein distance is a metric to describe the distance between the distributions of 2 datasets. Formula:
+
+
+W = ({\int_0^1}{{|{F_A}^{-1}(u) - {F_B}^{-1}(u)|}^2 du} )^{0.5}
+
+
+
where:
+
+W F_A , F_B {F_A}^{-1} , {F_B}^{-1}
+Jensen–Shannon divergence
+The Jensen–Shannon divergence is a method of measuring the similarity between two probability distributions. Formula:
+
+
+JS = 1/2 * KL(P || M) + 1/2 * KL(Q || M)
+
+
+
where:
+
+JS KL – sum x X P(x) log(Q(x) / P(x)) P,Q M {1 \over 2} * (P+Q)
+Machine learning models
+Light Gradient Boosting Machine
+LightGBM is an open-source framework for gradient boosted machines. By default LightGBM will train a Gradient Boosted Decision Tree (GBDT), but it also supports random forests, Dropouts meet Multiple Additive Regression Trees (DART), and Gradient Based One-Side Sampling (Goss). The framework is fast and was designed for distributed training. It supports large-scale datasets and training on the GPU. LightGBM also provide highly optimised, scalable and fast implementations of gradient boosted machines (GBMs). The official documentation of LightGBM is accessible here .
+Explainable AI models
+Local Interpretable Model-agnostic Explanations
+LIME (Local Interpretable Model-agnostic Explanations), an explainable AI technique, aids in illuminating a machine learning model and making each prediction's particular implications understandable. The technique is appropriate for local explanations since it describes the classifier for a particular single instance. LIME modifies the input data to produce a succession of false data that only partially retain the original features. The original implementation along with documentation of LIME technique could be found in this repo .
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Understanding machine learning monitoring - Whitebox
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Understanding machine learning monitoring
+Machine learning (ML) monitoring is a crucial process for ensuring the performance and reliability of ML models and systems. It involves tracking metrics, identifying issues, and improving overall performance. In this article, we will explore the key aspects of ML monitoring and its importance in today's data-driven world.
+What is machine learning monitoring?
+ML monitoring is the process of tracking the performance and behavior of ML models and systems. This includes monitoring metrics such as accuracy, precision, recall, and model performance over time. By monitoring these metrics, organizations can identify when a model is performing poorly or behaving unexpectedly and take action to correct it. Additionally, ML monitoring can be used to track the performance of different models and compare them to identify which model is performing best.
+
+One of the key aspects of ML monitoring is being able to detect when a model is performing poorly or behaving unexpectedly. This can be done by setting up alerts on certain metrics or by monitoring for unexpected changes in the model's behavior. For example, if a model's accuracy suddenly drops, an alert can be triggered to notify the team responsible for maintaining the model. This allows them to quickly investigate and address the issue, minimizing the impact on the organization's operations.
+Monitoring data and concept drift
+Another important aspect of ML monitoring is the ability to detect and address data and concept drifting. Data drifting refers to the gradual change in the distribution or characteristics of the input data over time. As the data changes, the model's performance may decrease, which is why it is important to detect data drifting and retrain the model with new data. Concept drifting, on the other hand, happens when the statistical properties of the target variable, which the model is trying to predict, change over time. This can happen due to various reasons such as changes in the underlying data distribution, overfitting, or degradation of the model's parameters. To address these issues, beside drift detection, organizations can use techniques such as monitoring of the performance of the model over time, retraining the model regularly etc.
+
+Another important aspect of ML monitoring is being able to track the input data that is being fed into the model. This can be used to identify any issues with the data, such as missing values, and to ensure that the data is being processed correctly. This helps to ensure that the model is making accurate predictions and that the data is being used effectively.
+Monitoring infrastructure and resources
+Finally, it is important to monitor the infrastructure and resources that the ML models are running on. This includes monitoring things like CPU and memory usage, disk space, and network traffic. This helps to ensure that the models have the resources they need to perform well and to identify any potential bottlenecks that may be impacting performance. By monitoring the infrastructure, organizations can ensure that their models are running smoothly and can make adjustments as needed.
+Conclusion
+In conclusion, ML monitoring is an essential process that helps organizations ensure the performance and reliability of their ML models and systems. By tracking metrics, identifying issues, and monitoring input data and infrastructure, organizations can ensure that their models are delivering accurate and reliable results and that their systems are running smoothly. With the increasing reliance on data and ML, the importance of ML monitoring will only continue to grow.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
SDK Documentation - Whitebox
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+SDK Documentation
+This is the documentation for Whitebox's SDK. For an interactive experience, you can expirement with the SDK's Jupyter notebooks .
+Models
+create_model (name, type, prediction, labels=None, description="")
+Creates a model in the database. This model works as placeholder for all the actual model's metadata.
+
+
+
+Parameter
+Type
+Description
+
+
+
+
+name strThe name of the model.
+
+
+type strThe model's type. Possible values: binary, multi_class, regression.
+
+
+prediction strThe prediction of the model.
+
+
+labels Dict[str, int]The model's labels. Defaults to None.
+
+
+description strThe model's description. Defaults to an empty string "".
+
+
+
+
+
Info
+
Labels are not applicable ONLY in regression models.
+
get_model (model_id)
+Fetches the model with the specified ID from the database.
+
+
+
+Parameter
+Type
+Description
+
+
+
+
+model_id strThe ID of the model.
+
+
+
+delete_model (model_id)
+Deletes the model with the specified ID from the database.
+
+
+
+Parameter
+Type
+Description
+
+
+
+
+model_id strThe ID of the model.
+
+
+
+Training Datasets
+log_training_dataset (model_id, non_processed, processed)
+Inserts a set of dataset rows into the database. When the dataset rows are successfully saved, the pipeline for training the model is triggered. Then, the trained model is saved in the /models/your_model's_id folder of whitebox's root directory.
+
+
+
+Parameter
+Type
+Description
+
+
+
+
+model_id strThe ID of the model.
+
+
+non_processed pd.DataFrameThe non processed training dataset.
+
+
+processed pd.DataFrameThe processed training dataset.
+
+
+
+
+
Info
+
The non processed and processed dataframes must have the same length.
+
Inferences
+log_inferences (model_id, non_processed, processed, timestamps, actuals=None)
+Inserts a set of inference rows into the database.
+
+
+
+Parameter
+Type
+Description
+
+
+
+
+model_id strThe ID of the model.
+
+
+non_processed pd.DataFrameThe non processed inferences.
+
+
+processed pd.DataFrameThe processed inferences.
+
+
+timestamps pd.SeriesThe timestamps for each inference row in the inference dataframes.
+
+
+actuals pd.SeriesThe actuals for each inference row in the inference dataframes. Defaults to None.
+
+
+
+
+
Info
+
The non processed and processed dataframes along with the timestamps and actuals series must ALL have the same length.
+
get_xai_row (inference_row_id)
+Produces an explainability report for a specific inference row.
+
+
+
+Parameter
+Type
+Description
+
+
+
+
+inference_row_id strThe ID of the inference row.
+
+
+
+Monitors
+create_model_monitor (model_id, name, status, metric, severity, email, feature=None, lower_threshold=None)
+Creates a monitor for a specific metric.
+
+
+
+Parameter
+Type
+Description
+
+
+
+
+model_id strThe ID of the model.
+
+
+name strThe name of the monitor.
+
+
+status MonitorStatusThe status of the monitor. Possible values for MonitorStatus: active, inactive.
+
+
+metric MonitorMetricsThe metric that will be monitored. Possible values for MonitorMetrics: accuracy, precision, recall, f1, r_square, mean_squared_error, mean_absolute_error, data_drift, concept_drift.
+
+
+severity AlertSeverityThe severity of the alert the monitor produces. Possible values for AlertSeverity: low, mid, high.
+
+
+email strThe email to which the alert will be sent.
+
+
+feature strThe feature to be monitored. Defaults to None.
+
+
+lower_threshold floatThe threshold below which an alert will be produced. Defaults to None.
+
+
+
+
+
Note
+
Some metrics like the data drift don't use a threshold so the feature that will be monitored should be inserted. In any case, both feature and lower_threshold can't be None at the same time.
+
Metrics
+get_drifting_metrics (model_id)
+Fetches a model's drifting metric reports.
+
+
+
+Parameter
+Type
+Description
+
+
+
+
+model_id strThe ID of the model.
+
+
+
+get_descriptive_statistics (model_id)
+Fetches a model's descriptive statistics reports.
+
+
+
+Parameter
+Type
+Description
+
+
+
+
+model_id strThe ID of the model.
+
+
+
+get_performance_metrics (model_id)
+Fetches a model's performance metric reports.
+
+
+
+Parameter
+Type
+Description
+
+
+
+
+model_id strThe ID of the model.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Whitebox is an open source E2E ML monitoring platform with edge capabilities that plays nicely with kubernetes
Documentation: https://whitebox-ai.github.io/whitebox/
Source Code: https://github.com/whitebox-ai/whitebox
Roadmap: https://github.com/whitebox-ai/whitebox/milestones
Issue tracking https://github.com/orgs/whitebox-ai/projects/1/views/3
Discord: https://discord.gg/bkAcsx4V
Whitebox is a dynamic open source E2E ML monitoring platform with edge capabilities that plays nicely with kubernetes.
The current key features are:
Descriptive statistics Classification models evaluation metrics Data / Concept drift monitoring Explainable AI Alerts Design guidelines:
Easy: Very easy to set up and get started with. Intuitive: Designed to be intuitive and easy to use. Pythonic SDK: Pythonic SDK for building your own monitoring infrastructure. Robust: Get production-ready MLOps system. Kubernetes: Get production-ready code. With automatic interactive documentation. "},{"location":"features/","title":"Features","text":""},{"location":"features/#design-decisions","title":"Design decisions","text":" Easy: Very easy to set up and get started with. Intuitive: Designed to be intuitive and easy to use. Pythonic SDK: Pythonic SDK for building your own monitoring infrastructure. Robust: Get production-ready MLOps system. Kubernetes: Get production-ready code. With automatic interactive documentation. "},{"location":"features/#descriptive-statistics","title":"Descriptive Statistics","text":"Whitebox provides a nice list of descriptive statistics of input dataset, making the overview of data easy.
"},{"location":"features/#models-metrics","title":"Models Metrics","text":""},{"location":"features/#classification-models","title":"Classification Models","text":"Whitebox includes comprehensive metrics tracking for classification models. This allows users to easily evaluate the performance of their classification models and identify areas for improvement. Additionally, users can set custom thresholds for each metric to receive alerts when performance deviates from expected results.
"},{"location":"features/#regression-models","title":"Regression Models","text":"Whitebox includes comprehensive metrics tracking for regression models. This allows users to easily evaluate the performance of their regression models and identify areas for improvement. Additionally, users can set custom thresholds for each metric to receive alerts when performance deviates from expected results.
"},{"location":"features/#data-concept-drift-monitoring","title":"Data / Concept Drift Monitoring","text":"Whitebox includes monitoring for data and concept drift. This feature tracks changes in the distribution of the data used to train models and alerts users when significant changes occur. Additionally, it detects changes in the performance of deployed models and alerts users when significant drift is detected. This allows users to identify and address data and model drift early, reducing the risk of poor model performance.
"},{"location":"features/#explainable-ai","title":"Explainable AI","text":"Whitebox includes model explaination also. The explainability performed through the explainability report which allows user to know anytime which feature had the most impact on model's prediction.
"},{"location":"features/#alerts","title":"Alerts","text":"Whitebox includes an alerting system that allows users to set custom thresholds for metrics and receive notifications when performance deviates from expected results. These alerts can be delivered via email, SMS, or webhook, and can be customized to fit the needs of any organization. This allows users to quickly respond to changes in model performance and take action to improve results.
"},{"location":"metric-definitions/","title":"Glossary / Metric definitions","text":""},{"location":"metric-definitions/#descriptive-statistics","title":"Descriptive statistics","text":""},{"location":"metric-definitions/#missing-values","title":"Missing values","text":"Missing values metric calculates the summary of the number of missing values per feature. Missing values include NaN in numeric arrays, NaN or None in object arrays and NaT in datetimelike.
"},{"location":"metric-definitions/#non-missing-values","title":"Non-Missing values","text":"Non-Missing values metric calculates the summary of the number of non-missing values per feature. Non-Missing values are all values beside NaN for numeric arrays, NaN or None for object arrays and NaT for datetimelike.
"},{"location":"metric-definitions/#mean-or-average-value","title":"Mean or Average value","text":"Returns the average value per feature excluding NaN and null values.
"},{"location":"metric-definitions/#minimum-value","title":"Minimum value","text":"Returns the minimum value per feature.
"},{"location":"metric-definitions/#maximum-value","title":"Maximum value","text":"Returns the maximum value per feature.
"},{"location":"metric-definitions/#summary","title":"Summary","text":"Returns the summary of the values per feature. Excludes NaN and null values during calculations.
"},{"location":"metric-definitions/#standard-deviation","title":"Standard Deviation","text":"Returns the sample standard deviation per feature normalized by N-1 excluding NaN and null values during calculations. Formula:
\u03c3 = \\sqrt{\u03a3(x_i-\u03bc)^2 \\over \u039d-1}"},{"location":"metric-definitions/#variance","title":"Variance","text":"Returns the unbiased variance per feature normalized by N-1 excluding NaN and null values during calculations. Formula:
\u03c3^2 = {\u03a3(x_i-\u03bc)^2 \\over \u039d-1} \u03c3^2 = {\u03a3(x_i-\u03bc)^2 \\over \u039d-1}"},{"location":"metric-definitions/#classification-evaluation-metrics","title":"Classification evaluation metrics","text":""},{"location":"metric-definitions/#confusion-matrix","title":"Confusion Matrix","text":"Returns the number of TP, TN, FP and FN. In case of a multi-class classification returns the number of TP, TN, FP and FN per class.
A typical example for binary classification could be seen below in which:
20 observations were correctly classified as positive. 10 observations were incorrectly classified as negative while they were actually positive. 5 observations were incorrectly classified as positive while they were actually negative. 75 observations were correctly classified as negative. Predicted Positive Predicted Negative Actual Positive 20 (TP) 10 (FN) Actual Negative 5 (FP) 75 (TN) A typical example for multi-class classification could be seen below in which:
15 observations were correctly classified as Class A. 5 observations were incorrectly classified as Class B while they were actually Class A. 2 observations were incorrectly classified as Class C while they were actually Class A. 4 observations were incorrectly classified as Class A while they were actually Class B. 20 observations were correctly classified as Class B. 3 observations were incorrectly classified as Class C while they were actually Class B. 2 observations were incorrectly classified as Class A while they were actually Class C. 8 observations were incorrectly classified as Class B while they were actually Class C. 25 observations were correctly classified as Class C. Predicted Class A Predicted Class B Predicted Class C Actual Class A 15 (TP_A) 5 2 Actual Class B 4 20 (TP_B) 3 Actual Class C 2 8 25 (TP_C)"},{"location":"metric-definitions/#accuracy","title":"Accuracy","text":"Returns the accuracy classification score. In multi-class classification, this function computes subset accuracy: the set of labels predicted for a sample must exactly match the corresponding set of labels in y_true. Formula:
accuracy = {(TP + TN) \\over (TP + TN + FP + FN)} accuracy = {(TP + TN) \\over (TP + TN + FP + FN)}"},{"location":"metric-definitions/#precision","title":"Precision","text":"Returns the precision classification score. In multi-class classification, returns the below 3 scores:
micro: Calculate metrics globally by counting the total true positives, false negatives and false positives.macro: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account.weighted: Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters \u2018macro\u2019 to account for label imbalance; it can result in an F-score that is not between precision and recall. Formula:
precision = {TP \\over (TP + FP)} precision = {TP \\over (TP + FP)}"},{"location":"metric-definitions/#recall","title":"Recall","text":"Returns the recall classification score. In multi-class classification, returns the below 3 scores:
micro: Calculate metrics globally by counting the total true positives, false negatives and false positives.macro: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account.weighted: Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters \u2018macro\u2019 to account for label imbalance; it can result in an F-score that is not between precision and recall. Formula:
recall = {TP \\over (TP + FN)} recall = {TP \\over (TP + FN)}"},{"location":"metric-definitions/#f1-score","title":"F1 score","text":"Returns the f1 classification score. In multi-class classification, returns the below 3 scores:
micro: Calculate metrics globally by counting the total true positives, false negatives and false positives.macro: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account.weighted: Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters \u2018macro\u2019 to account for label imbalance; it can result in an F-score that is not between precision and recall. Formula:
F1 = 2 * {(precision * recall) \\over (precision + recall)} F1 = 2 * {(precision * recall) \\over (precision + recall)}"},{"location":"metric-definitions/#regression-evaluation-metrics","title":"Regression evaluation metrics","text":""},{"location":"metric-definitions/#r-squared","title":"R-Squared","text":"In a regression model, R-Squared (also known as R2 or the coefficient of determination) is a statistical metric that quantifies how much of the variance in the dependent variable can be accounted for by the independent variable. R-squared, thus, displays how well the data match the regression model (the goodness of fit).
Formula:
R^2 = {SS_{regression} \\over SS_{total}} R^2 = {SS_{regression} \\over SS_{total}} where:
SS_{regression}SS_{regression} = sum of squares due to regression (explained sum of squares) SS_{total}SS_{total} = total sum of squares "},{"location":"metric-definitions/#mean-squared-error","title":"Mean squared error","text":"The degree of inaccuracy in statistical models is gauged by the mean squared error, or MSE. Between the observed and projected values, it evaluates the average squared difference. The MSE is equal to 0 when a model is error-free. As model error increases, its value increases.
Formula:
MSE = {\u03a3(y_{i}-y'_{i})^2 \\over n} MSE = {\u03a3(y_{i}-y'_{i})^2 \\over n} where:
y_{i}y_{i} = i_{th}i_{th} observation y'_{i}y'_{i} = corresponding predicted value nn = number of observations "},{"location":"metric-definitions/#mean-absolute-error","title":"Mean absolute error","text":"The mean absolute error of a model with respect to a test set is the mean of the absolute values of the individual prediction errors on over all instances in the test set. Each prediction error is the difference between the true value and the predicted value for the instance.
Formula:
MAE = {{{\u03a3^2}_{i=1}}|y_{i}-x_{i}| \\over n} MAE = {{{\u03a3^2}_{i=1}}|y_{i}-x_{i}| \\over n} where:
y_{i}y_{i} = prediction x_{i}x_{i} = true value nn = number of observations "},{"location":"metric-definitions/#statistical-tests-and-techniques","title":"Statistical tests and techniques","text":""},{"location":"metric-definitions/#kolmogorov-smirnov-two-sample-test","title":"Kolmogorov-Smirnov Two Sample test","text":"When there are two datasets then K-S two sample test can be used to test the agreement between their distributions. The null hypothesis states that there is no difference between the two distributions. Formula:
D = Max|{F_a(X)-F_b(X)}| D = Max|{F_a(X)-F_b(X)}| where:
aa = observations from first dataset. bb = observations from second dataset. F_n(X)F_n(X) = observed cumulative frequency distribution of a random sample of n observations. "},{"location":"metric-definitions/#chi-squared-test","title":"Chi-squared test","text":"A chi-square test is a statistical test used to compare 2 datasets. The purpose of this test is to determine if a difference between data of 2 datasets is due to chance, or if it is due to a relationship between the variables you are studying. Formula:
x^2 = \u03a3{(O_i - E_i)^2 \\over E_i} x^2 = \u03a3{(O_i - E_i)^2 \\over E_i} where:
x^2x^2 = chi-square O_iO_i = 1st dataset values E_iE_i = 2nd dataset values "},{"location":"metric-definitions/#z-score-for-independent-proportions","title":"Z-score for independent proportions","text":"The purpose of the z-test for independent proportions is to compare two independent datasets. Formula:
Z = {p_1 - p_2 \\over \\sqrt{p' q' ({1\\over n_1} + {1\\over n_2})}} Z = {p_1 - p_2 \\over \\sqrt{p' q' ({1\\over n_1} + {1\\over n_2})}} where:
ZZ = Z-statistic which is compared to the standard normal deviate p_1 , p_2p_1 , p_2 = two datasets proportions p'p' = estimated true proportion under the null hypothesis q'q' = (1-p')(1-p') n_1 , n_2n_1 , n_2 = number of observations in two datasets "},{"location":"metric-definitions/#wasserstein-distance","title":"Wasserstein distance","text":"The Wasserstein distance is a metric to describe the distance between the distributions of 2 datasets. Formula:
W = ({\\int_0^1}{{|{F_A}^{-1}(u) - {F_B}^{-1}(u)|}^2 du} )^{0.5} W = ({\\int_0^1}{{|{F_A}^{-1}(u) - {F_B}^{-1}(u)|}^2 du} )^{0.5} where:
WW = Wasserstein distance F_A , F_BF_A , F_B = corresponding cumulative distribution functions of two datasets {F_A}^{-1} , {F_B}^{-1}{F_A}^{-1} , {F_B}^{-1} = respective quantile functions "},{"location":"metric-definitions/#jensenshannon-divergence","title":"Jensen\u2013Shannon divergence","text":"The Jensen\u2013Shannon divergence is a method of measuring the similarity between two probability distributions. Formula:
JS = 1/2 * KL(P || M) + 1/2 * KL(Q || M) JS = 1/2 * KL(P || M) + 1/2 * KL(Q || M) where:
JSJS = Jensen\u2013Shannon divergence KLKL = Kullback-Leibler divergence: \u2013 sum x\u2013 sum x in XX P(x)P(x) * log(Q(x) / P(x))log(Q(x) / P(x)) P,QP,Q = distributions of 2 datasets MM = {1 \\over 2} * (P+Q){1 \\over 2} * (P+Q) "},{"location":"metric-definitions/#machine-learning-models","title":"Machine learning models","text":""},{"location":"metric-definitions/#light-gradient-boosting-machine","title":"Light Gradient Boosting Machine","text":"LightGBM is an open-source framework for gradient boosted machines. By default LightGBM will train a Gradient Boosted Decision Tree (GBDT), but it also supports random forests, Dropouts meet Multiple Additive Regression Trees (DART), and Gradient Based One-Side Sampling (Goss). The framework is fast and was designed for distributed training. It supports large-scale datasets and training on the GPU. LightGBM also provide highly optimised, scalable and fast implementations of gradient boosted machines (GBMs). The official documentation of LightGBM is accessible here.
"},{"location":"metric-definitions/#explainable-ai-models","title":"Explainable AI models","text":""},{"location":"metric-definitions/#local-interpretable-model-agnostic-explanations","title":"Local Interpretable Model-agnostic Explanations","text":"LIME (Local Interpretable Model-agnostic Explanations), an explainable AI technique, aids in illuminating a machine learning model and making each prediction's particular implications understandable. The technique is appropriate for local explanations since it describes the classifier for a particular single instance. LIME modifies the input data to produce a succession of false data that only partially retain the original features. The original implementation along with documentation of LIME technique could be found in this repo.
"},{"location":"ml-monitoring/","title":"Understanding machine learning monitoring","text":"Machine learning (ML) monitoring is a crucial process for ensuring the performance and reliability of ML models and systems. It involves tracking metrics, identifying issues, and improving overall performance. In this article, we will explore the key aspects of ML monitoring and its importance in today's data-driven world.
"},{"location":"ml-monitoring/#what-is-machine-learning-monitoring","title":"What is machine learning monitoring?","text":"ML monitoring is the process of tracking the performance and behavior of ML models and systems. This includes monitoring metrics such as accuracy, precision, recall, and model performance over time. By monitoring these metrics, organizations can identify when a model is performing poorly or behaving unexpectedly and take action to correct it. Additionally, ML monitoring can be used to track the performance of different models and compare them to identify which model is performing best.
"},{"location":"ml-monitoring/#detecting-poor-performance","title":"Detecting poor performance","text":"One of the key aspects of ML monitoring is being able to detect when a model is performing poorly or behaving unexpectedly. This can be done by setting up alerts on certain metrics or by monitoring for unexpected changes in the model's behavior. For example, if a model's accuracy suddenly drops, an alert can be triggered to notify the team responsible for maintaining the model. This allows them to quickly investigate and address the issue, minimizing the impact on the organization's operations.
"},{"location":"ml-monitoring/#monitoring-data-and-concept-drift","title":"Monitoring data and concept drift","text":"Another important aspect of ML monitoring is the ability to detect and address data and concept drifting. Data drifting refers to the gradual change in the distribution or characteristics of the input data over time. As the data changes, the model's performance may decrease, which is why it is important to detect data drifting and retrain the model with new data. Concept drifting, on the other hand, happens when the statistical properties of the target variable, which the model is trying to predict, change over time. This can happen due to various reasons such as changes in the underlying data distribution, overfitting, or degradation of the model's parameters. To address these issues, beside drift detection, organizations can use techniques such as monitoring of the performance of the model over time, retraining the model regularly etc.
"},{"location":"ml-monitoring/#monitoring-input-data","title":"Monitoring input data","text":"Another important aspect of ML monitoring is being able to track the input data that is being fed into the model. This can be used to identify any issues with the data, such as missing values, and to ensure that the data is being processed correctly. This helps to ensure that the model is making accurate predictions and that the data is being used effectively.
"},{"location":"ml-monitoring/#monitoring-infrastructure-and-resources","title":"Monitoring infrastructure and resources","text":"Finally, it is important to monitor the infrastructure and resources that the ML models are running on. This includes monitoring things like CPU and memory usage, disk space, and network traffic. This helps to ensure that the models have the resources they need to perform well and to identify any potential bottlenecks that may be impacting performance. By monitoring the infrastructure, organizations can ensure that their models are running smoothly and can make adjustments as needed.
"},{"location":"ml-monitoring/#conclusion","title":"Conclusion","text":"In conclusion, ML monitoring is an essential process that helps organizations ensure the performance and reliability of their ML models and systems. By tracking metrics, identifying issues, and monitoring input data and infrastructure, organizations can ensure that their models are delivering accurate and reliable results and that their systems are running smoothly. With the increasing reliance on data and ML, the importance of ML monitoring will only continue to grow.
"},{"location":"sdk-docs/","title":"SDK Documentation","text":"This is the documentation for Whitebox's SDK. For an interactive experience, you can expirement with the SDK's Jupyter notebooks.
"},{"location":"sdk-docs/#models","title":"Models","text":"create_model(name, type, prediction, labels=None, description=\"\")
Creates a model in the database. This model works as placeholder for all the actual model's metadata.
Parameter Type Description name str The name of the model. type str The model's type. Possible values: binary, multi_class, regression. prediction str The prediction of the model. labels Dict[str, int] The model's labels. Defaults to None. description str The model's description. Defaults to an empty string \"\". Info
Labels are not applicable ONLY in regression models.
get_model(model_id)
Fetches the model with the specified ID from the database.
Parameter Type Description model_id str The ID of the model. delete_model(model_id)
Deletes the model with the specified ID from the database.
Parameter Type Description model_id str The ID of the model."},{"location":"sdk-docs/#training-datasets","title":"Training Datasets","text":"log_training_dataset(model_id, non_processed, processed)
Inserts a set of dataset rows into the database. When the dataset rows are successfully saved, the pipeline for training the model is triggered. Then, the trained model is saved in the /models/your_model's_id folder of whitebox's root directory.
Parameter Type Description model_id str The ID of the model. non_processed pd.DataFrame The non processed training dataset. processed pd.DataFrame The processed training dataset. Info
The non processed and processed dataframes must have the same length.
"},{"location":"sdk-docs/#inferences","title":"Inferences","text":"log_inferences(model_id, non_processed, processed, timestamps, actuals=None)
Inserts a set of inference rows into the database.
Parameter Type Description model_id str The ID of the model. non_processed pd.DataFrame The non processed inferences. processed pd.DataFrame The processed inferences. timestamps pd.Series The timestamps for each inference row in the inference dataframes. actuals pd.Series The actuals for each inference row in the inference dataframes. Defaults to None. Info
The non processed and processed dataframes along with the timestamps and actuals series must ALL have the same length.
get_xai_row(inference_row_id)
Produces an explainability report for a specific inference row.
Parameter Type Description inference_row_id str The ID of the inference row."},{"location":"sdk-docs/#monitors","title":"Monitors","text":"create_model_monitor(model_id, name, status, metric, severity, email, feature=None, lower_threshold=None)
Creates a monitor for a specific metric.
Parameter Type Description model_id str The ID of the model. name str The name of the monitor. status MonitorStatus The status of the monitor. Possible values for MonitorStatus: active, inactive. metric MonitorMetrics The metric that will be monitored. Possible values for MonitorMetrics: accuracy, precision, recall, f1, r_square, mean_squared_error, mean_absolute_error, data_drift, concept_drift. severity AlertSeverity The severity of the alert the monitor produces. Possible values for AlertSeverity: low, mid, high. email str The email to which the alert will be sent. feature str The feature to be monitored. Defaults to None. lower_threshold float The threshold below which an alert will be produced. Defaults to None. Note
Some metrics like the data drift don't use a threshold so the feature that will be monitored should be inserted. In any case, both feature and lower_threshold can't be None at the same time.
"},{"location":"sdk-docs/#metrics","title":"Metrics","text":"get_drifting_metrics(model_id)
Fetches a model's drifting metric reports.
Parameter Type Description model_id str The ID of the model. get_descriptive_statistics(model_id)
Fetches a model's descriptive statistics reports.
Parameter Type Description model_id str The ID of the model. get_performance_metrics(model_id)
Fetches a model's performance metric reports.
Parameter Type Description model_id str The ID of the model."},{"location":"tutorial/installation/","title":"Installation","text":""},{"location":"tutorial/installation/#docker","title":"Docker","text":"Install whitebox server and all of its dependencies using docker-compose
Copy the following code in a file named docker-compose.yml:
version: \"3.10\"\nname: Whitebox\nservices:\npostgres:\nimage: postgres:15\nrestart: unless-stopped\nenvironment:\n- POSTGRES_USER=postgres\n- POSTGRES_PASSWORD=postgres\n- POSTGRES_MULTIPLE_DATABASES=test # postgres db is created by default\nlogging:\noptions:\nmax-size: 10m\nmax-file: \"3\"\nports:\n- \"5432:5432\"\nvolumes:\n- wb_data:/var/lib/postgresql/data\nnetworks:\n- whitebox\nwhitebox:\nimage: sqdhub/whitebox:main\nrestart: unless-stopped\nenvironment:\n- APP_NAME=Whitebox | Docker\n- DATABASE_URL=postgresql://postgres:postgres@postgres:5432/postgres\n- SECRET_KEY=<add_your_own> # Optional, if not set the API key won't be encrypted\nports:\n- \"8000:8000\"\ndepends_on:\n- postgres\nnetworks:\n- whitebox\nvolumes:\nwb_data:\nnetworks:\nwhitebox:\nname: whitebox\nWith your terminal navigate to docker-compose.yml's location and then run the following command:
$ docker-compose up\nYou can also install Whitebox server and all of its dependencies in your k8s cluster using helm:
helm repo add squaredev https://chartmuseum.squaredev.io/\nhelm repo update\nhelm install whitebox squaredev/whitebox\nInfo
The metrics are automatically calculated in a set interval for all models in the database.
Depending on the data you have provided, the metrics calculation process will try to as many metrics possible for the specific model.
"},{"location":"tutorial/metrics/#metric-requirements","title":"Metric Requirements","text":"All metrics require that inferences are provided for the specific model. If not, the whole process is skipped. Each metric though has different requirements that need to be fulfilled in order to be calculated. The requirements are listed in the following sections.
"},{"location":"tutorial/metrics/#descriptive-statistics","title":"Descriptive Statistics","text":"The descriptive statistics are calculated per feature on any given dataset.
Metric Type of data missing_count numerical & categorical non_missing_count numerical & categorical mean numerical minimum numerical maximum numerical sum numerical standard_deviation numerical variance numerical"},{"location":"tutorial/metrics/#drifting-metrics","title":"Drifting Metrics","text":"The target of drift metrics is to calculate the data drift between 2 datasets. Currently supported drift types are:
Requirements:
Inferences Training Dataset Note
If actuals aren't provided for all inferences, then ONLY the inferences that have actuals will be used for the calculation of the drifting metrics.
"},{"location":"tutorial/metrics/#data-drift","title":"Data drift","text":"An analysis happens comparing the current data to the reference data estimating the distributions of each feature in the two datasets. The schema of both datasets should be identical.
Returns a drift summary of the following form:
{\n\"timestamp\": \"the timestamp of the report\",\n\"drift_summary\": {\n\"number_of_columns\": \"total number of dataset columns\",\n\"number_of_drifted_columns\": \"total number of drifted columns\",\n\"share_of_drifted_columns\": \"number_of_drifted_columns/number_of_columns\",\n\"dataset_drift\": \"Boolean based on the criteria below\",\n\"drift_by_columns\": {\n\"column1\": {\n\"column_name\": \"column1\",\n\"column_type\": \"the type of column (e.g. num)\",\n\"stattest_name\": \"the statistical test tha was used\",\n\"drift_score\": \"the drifting score based on the test\",\n\"drift_detected\": \"Boolean based on the criteria below\",\n\"threshold\": \"a float number based on the criteria below\"\n},\n\"column2\": {\"...\"}\n}\n}\n}\nLogic to choose the appropriate statistical test is based on:
feature type: categorical or numerical the number of observations in the reference dataset the number of unique values in the feature (n_unique) For small data with <= 1000 observations in the reference dataset:
For numerical features (n_unique > 5): two-sample Kolmogorov-Smirnov test. For categorical features or numerical features with n_unique <= 5: chi-squared test. For binary categorical features (n_unique <= 2), we use the proportion difference test for independent samples based on Z-score. All tests use a 0.95 confidence level by default.
For larger data with > 1000 observations in the reference dataset:
For numerical features (n_unique > 5): Wasserstein Distance. For categorical features or numerical with n_unique <= 5): Jensen\u2013Shannon divergence. All tests use a threshold = 0.1 by default.
"},{"location":"tutorial/metrics/#concept-drift","title":"Concept drift","text":"An analysis happens comparing the current target feature to the reference target feature.
Returns a concept drift summary of the following form:
{\n\"timestamp\": \"the timestamp of the report\",\n\"concept_drift_summary\": {\n\"column_name\": \"column1\",\n\"column_type\": \"the type of column (e.g. num)\",\n\"stattest_name\": \"the statistical test tha was used\",\n\"threshold\": \"threshold used based on criteria below\",\n\"drift_score\": \"the drifting score based on the test\",\n\"drift_detected\": \"Boolean based on the criteria below,\"\n}\n}\nLogic to choose the appropriate statistical test is based on:
the number of observations in the reference dataset the number of unique values in the target (n_unique) For small data with <= 1000 observations in the reference dataset:
For categorical target with n_unique > 2: chi-squared test. For binary categorical target (n_unique <= 2), we use the proportion difference test for independent samples based on Z-score. All tests use a 0.95 confidence level by default.
For larger data with > 1000 observations in the reference dataset we use Jensen\u2013Shannon divergence with a threshold = 0.1 .
"},{"location":"tutorial/metrics/#performance-evaluation-metrics","title":"Performance Evaluation Metrics","text":"Requirements:
Inferences Actuals for the inferences The target of evaluation metrics is to evaluate the quality of an machine learning model. Currently supported models are:
Binary classification Multi-class classification Regression Metric Supported model confusion matrix binary classification & multi-class classification accuracy binary classification & multi-class classification precision binary classification & multi-class classification recall binary classification & multi-class classification f1 score binary classification & multi-class classification r_square regression mean_squared_error regression mean_absolute_error regression"},{"location":"tutorial/metrics/#explainability","title":"Explainability","text":"Requirements:
The target of the explainability feature is to provide a contribution score of each feature to each individual prediction, in a try to explain how the model concluded in the specific prediction. To achieve this we define 3 Levels of confidence in the explainability feature, based on how accessible or not is the client's input:
Level-0: In this level a replacement model is trained in the same training data as client's model, and used for the explainability feature. Level-1 (pending): In this level a surrogate model is trained in a way of trying to achieve the same predictions as client's model, and used for the explainability feature. Level-2 (pending): Client's model is used for the explainability feature. "},{"location":"tutorial/metrics/#level-0-confidence","title":"Level-0 confidence","text":"At this level a replacement model is trained in the same training data as client's model, and used for the explainability feature. Below there is a table of used models per machine learning task.
Model Task LightGBM binary classification & multi-class classification & regression The fine tuning of models, through a hyper-parameters exploration is a pending task for now.
The trained model along with the inference data are used as input in LIME library, and the below report is provided - presenting the contribution of each feature in a specific prediction (the report includes all the feature contribution score to an descending order based on the absolute score value):
{\n\"feature1\": \"contribution score\",\n\"feature2\": \"contribution score\",\n\"feature3\": \"contribution score\"\n}\nComing soon
"},{"location":"tutorial/metrics/#level-2-confidence","title":"Level-2 confidence","text":"Coming soon
"},{"location":"tutorial/monitors_alerts/","title":"Monitors and Alerts","text":""},{"location":"tutorial/monitors_alerts/#monitors","title":"Monitors","text":"You can create a monitor in whitebox so that alert are created automaticaly when some value is out of bounds. Here is an example:
from whitebox import Whitebox, MonitorStatus, MonitorMetrics, AlertSeverity\nwb = Whitebox(host=\"127.0.0.1:8000\", api_key=\"some_api_key\")\nmodel_monitor = wb.create_model_monitor(\nmodel_id=\"mock_model_id\",\nname=\"test\",\nstatus=MonitorStatus.active,\nmetric=MonitorMetrics.accuracy,\nseverity=AlertSeverity.high,\nemail=\"jackie.chan@somemail.io\",\nlower_threshold=0.7\n)\nOnce the metrics reports have been produced, the monitoring alert pipeline is triggered. This means that if you have created any model monitors for a specific metric, alerts will be created if certain criteria are met, based on the thresholds and the monitor types you have specified.
"},{"location":"tutorial/sdk/","title":"Using Whitebox SDK","text":""},{"location":"tutorial/sdk/#installation","title":"Installation","text":"Installing Whitebox is a pretty easy job. Just install it like any other python package.
Install the SDK with pip:
$ pip install whitebox-sdk\nAll the required packages will be automatically installed!
Now you're good to go!
"},{"location":"tutorial/sdk/#initial-setup","title":"Initial Setup","text":"In order to run Whitebox, you will need the application's API key. This key will be produced for you during the initial run of the Uvicorn live server. Assuming you run the server with docker compose (you can find more in the install page of the tutorial), you will see the following output:
$ docker compose up\n\n...\n<span style=\"color: green;\">INFO</span>: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit)\n<span style=\"color: green;\">INFO</span>: Started reloader process [4450] using StatReload\n<span style=\"color: green;\">INFO</span>: Started server process [4452]\n<span style=\"color: green;\">INFO</span>: Waiting for application startup.\n<span style=\"color: green;\">INFO</span>: Created username: admin, API key: some_api_key\n<span style=\"color: green;\">INFO</span>: Application startup complete.\n...\nInfo
Keep this API key somewhere safe!
If you lose it, you will need to delete the admin user in your database and re-run the live serve to produce a new key!
After you get the API key, all you have to do is create an instance of the Whitebox class adding your host and API key as parameters:
from whitebox import Whitebox\nwb = Whitebox(host=\"http://127.0.0.1:8000\", api_key=\"some_api_key\")\nNow you're ready to start using Whitebox!
"},{"location":"tutorial/sdk/#models","title":"Models","text":""},{"location":"tutorial/sdk/#creating-a-model","title":"Creating a Model","text":"In order to start adding training datasets and inferences, you first need to create a model.
Let's create a sample model:
wb.create_model(\nname=\"Model 1\",\ntype=\"binary\",\nlabels={\n'additionalProp1': 0,\n'additionalProp2': 1\n},\nprediction=\"target\"\n)\nFor more details about the schema accepted property types visit the Models section in the SDK documentation.
"},{"location":"tutorial/sdk/#fetching-a-model","title":"Fetching a Model","text":"Getting a model from the database is as easy as it sounds. You'll just need the model_id:
wb.get_model(\"some_model_id\")\nDeleting a model is as easy as fetching a model. Just use the model_id:
wb.delete_model(\"some_model_id\")\nWarning
You will have to be extra careful when deleting a model because all datasets, inferences, monitors and literally everything will be deleted from the database along with the model itself!
"},{"location":"tutorial/sdk/#loading-training-datasets","title":"Loading Training Datasets","text":"Once you have created a model you can start loading your data. Let's start with the training dataset!
In our example we will create a pd.DataFrame from a .csv file. Of course you can use any method you like to create your pd.DataFrame as long as your non-processed and processed datasets have the same amount of rows (a.k.a. the same length) and there are more than one rows!
import pandas as pd\nnon_processed_df = pd.read_csv(\"path/to/file/non_processed_data.csv\")\nprocessed_df = pd.read_csv(\"path/to/file/processed_data.csv\")\nwb.log_training_dataset(\nmodel_id=\"some_model_id\",\nnon_processed=non_processed_df,\nprocessed=processed_df\n)\nNote
When your training dataset is saved in the database, the model training process will begin excecuting, based on this dataset and the model it's associated with. That's why you need to load all the rows of your training dataset in the same batch.
"},{"location":"tutorial/sdk/#loading-inferences","title":"Loading Inferences","text":"To load your inferences you have to follow the exact same procedure as with the training datasets. The only difference is that you need to provide a pd.Series with the timestamps and (optionally) a pd.Series with the actuals, whose indices should match the ones in the non-processed and processed pd.DataFrames.
In our example let's assume that both the non-processed and processed pd.DataFrames have 10 rows each:
import pandas as pd\nnon_processed_df = pd.read_csv(\"path/to/file/non_processed_data.csv\")\nprocessed_df = pd.read_csv(\"path/to/file/processed_data.csv\")\n# Timestamps and actuals should have a length of 10\ntimestamps = pd.Series([\"2022-12-22T12:13:27.879738\"] * 10)\nactuals = pd.Series([0, 1, 1, 1, 0, 0, 1, 1, 0, 0])\nwb.log_inferences(\nmodel_id=\"some_model_id\",\nnon_processed=non_processed_df,\nprocessed=processed_df,\ntimestamps=timestamps,\nactuals=actuals\n)\nWarning
Make sure you add the actuals if you already know them, because as of now, there's no ability to add them at a later time by updating the inference rows.
"}]}
\ No newline at end of file
diff --git a/setup.py b/setup.py
deleted file mode 100644
index cf39713..0000000
--- a/setup.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from setuptools import setup, find_packages
-from pathlib import Path
-
-
-VERSION = "0.0.20"
-
-DESCRIPTION = "Whitebox is an open source E2E ML monitoring platform with edge capabilities that plays nicely with kubernetes"
-LONG_DESCRIPTION = (Path(__file__).parent / "README.md").read_text()
-
-# Setting up
-setup(
- # the name must match the folder name 'verysimplemodule'
- name="whitebox-sdk",
- version=VERSION,
- author="Squaredev",
- author_email="hello@squaredev.io",
- description=DESCRIPTION,
- long_description=LONG_DESCRIPTION, # add README.md
- long_description_content_type="text/markdown",
- packages=find_packages(),
- install_requires=[], # add any additional packages that
- # needs to be installed along with your package. Eg: 'caer'
- keywords=["python", "model monitoring", "whitebox", "mlops"],
- classifiers=[
- "Programming Language :: Python :: 3",
- ],
-)
diff --git a/sitemap.xml b/sitemap.xml
new file mode 100644
index 0000000..4f747da
--- /dev/null
+++ b/sitemap.xml
@@ -0,0 +1,48 @@
+
+
+
+ None
+ 2023-02-23
+ daily
+
+
+ None
+ 2023-02-23
+ daily
+
+
+ None
+ 2023-02-23
+ daily
+
+
+ None
+ 2023-02-23
+ daily
+
+
+ None
+ 2023-02-23
+ daily
+
+
+ None
+ 2023-02-23
+ daily
+
+
+ None
+ 2023-02-23
+ daily
+
+
+ None
+ 2023-02-23
+ daily
+
+
+ None
+ 2023-02-23
+ daily
+
+
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
new file mode 100644
index 0000000..c7ed163
Binary files /dev/null and b/sitemap.xml.gz differ
diff --git a/tutorial/installation/index.html b/tutorial/installation/index.html
new file mode 100644
index 0000000..19169d5
--- /dev/null
+++ b/tutorial/installation/index.html
@@ -0,0 +1,724 @@
+
+
+
+
+
+ Installation - Whitebox
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Installation
+Docker
+Install whitebox server and all of its dependencies using docker-compose
+Copy the following code in a file named docker-compose.yml:
+version : "3.10"
+name : Whitebox
+services :
+ postgres :
+ image : postgres:15
+ restart : unless-stopped
+ environment :
+ - POSTGRES_USER=postgres
+ - POSTGRES_PASSWORD=postgres
+ - POSTGRES_MULTIPLE_DATABASES=test # postgres db is created by default
+ logging :
+ options :
+ max-size : 10m
+ max-file : "3"
+ ports :
+ - "5432:5432"
+ volumes :
+ - wb_data:/var/lib/postgresql/data
+ networks :
+ - whitebox
+ whitebox :
+ image : sqdhub/whitebox:main
+ restart : unless-stopped
+ environment :
+ - APP_NAME=Whitebox | Docker
+ - DATABASE_URL=postgresql://postgres:postgres@postgres:5432/postgres
+ - SECRET_KEY=<add_your_own> # Optional, if not set the API key won't be encrypted
+ ports :
+ - "8000:8000"
+ depends_on :
+ - postgres
+ networks :
+ - whitebox
+volumes :
+ wb_data :
+networks :
+ whitebox :
+ name : whitebox
+With your terminal navigate to docker-compose.yml's location and then run the following command:
+
+
+Kubernetes
+You can also install Whitebox server and all of its dependencies in your k8s cluster using helm:
+ repo add squaredev https://chartmuseum.squaredev.io/
+ repo update
+ install whitebox squaredev/whitebox
+
+
+
+
+
+
+
+
+
Metrics Calculation - Whitebox
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Metrics Calculation
+
+
Info
+
The metrics are automatically calculated in a set interval for all models in the database.
+
Depending on the data you have provided, the metrics calculation process will try to as many metrics possible for the specific model.
+
Metric Requirements
+All metrics require that inferences are provided for the specific model. If not, the whole process is skipped.
+Each metric though has different requirements that need to be fulfilled in order to be calculated.
+The requirements are listed in the following sections.
+Descriptive Statistics
+The descriptive statistics are calculated per feature on any given dataset.
+
+
+
+Metric
+Type of data
+
+
+
+
+missing_count numerical & categorical
+
+non_missing_count numerical & categorical
+
+mean numerical
+
+minimum numerical
+
+maximum numerical
+
+sum numerical
+
+standard_deviation numerical
+
+variance numerical
+
+
+Drifting Metrics
+The target of drift metrics is to calculate the data drift between 2 datasets. Currently supported drift types are:
+
+Data drift
+Concept drift
+
+Requirements :
+
+Inferences
+Training Dataset
+
+
+
Note
+
If actuals aren't provided for all inferences, then ONLY the inferences that have actuals will be used for the calculation of the drifting metrics.
+
Data drift
+An analysis happens comparing the current data to the reference data estimating the distributions of each feature in the two datasets. The schema of both datasets should be identical.
+Returns a drift summary of the following form:
+{
+ "timestamp" : "the timestamp of the report" ,
+ "drift_summary" : {
+ "number_of_columns" : "total number of dataset columns" ,
+ "number_of_drifted_columns" : "total number of drifted columns" ,
+ "share_of_drifted_columns" : "number_of_drifted_columns/number_of_columns" ,
+ "dataset_drift" : "Boolean based on the criteria below" ,
+ "drift_by_columns" : {
+ "column1" : {
+ "column_name" : "column1" ,
+ "column_type" : "the type of column (e.g. num)" ,
+ "stattest_name" : "the statistical test tha was used" ,
+ "drift_score" : "the drifting score based on the test" ,
+ "drift_detected" : "Boolean based on the criteria below" ,
+ "threshold" : "a float number based on the criteria below"
+ },
+ "column2" : { "..." }
+ }
+ }
+}
+Logic to choose the appropriate statistical test is based on:
+
+feature type: categorical or numerical
+the number of observations in the reference dataset
+the number of unique values in the feature (n_unique)
+
+For small data with <= 1000 observations in the reference dataset:
+
+For numerical features (n_unique > 5): two-sample Kolmogorov-Smirnov test.
+For categorical features or numerical features with n_unique <= 5: chi-squared test.
+For binary categorical features (n_unique <= 2), we use the proportion difference test for independent samples based on Z-score.
+
+All tests use a 0.95 confidence level by default.
+For larger data with > 1000 observations in the reference dataset:
+
+For numerical features (n_unique > 5): Wasserstein Distance.
+For categorical features or numerical with n_unique <= 5): Jensen–Shannon divergence.
+
+All tests use a threshold = 0.1 by default.
+Concept drift
+An analysis happens comparing the current target feature to the reference target feature.
+Returns a concept drift summary of the following form:
+{
+ "timestamp" : "the timestamp of the report" ,
+ "concept_drift_summary" : {
+ "column_name" : "column1" ,
+ "column_type" : "the type of column (e.g. num)" ,
+ "stattest_name" : "the statistical test tha was used" ,
+ "threshold" : "threshold used based on criteria below" ,
+ "drift_score" : "the drifting score based on the test" ,
+ "drift_detected" : "Boolean based on the criteria below,"
+ }
+}
+Logic to choose the appropriate statistical test is based on:
+
+the number of observations in the reference dataset
+the number of unique values in the target (n_unique)
+
+For small data with <= 1000 observations in the reference dataset:
+
+For categorical target with n_unique > 2: chi-squared test.
+For binary categorical target (n_unique <= 2), we use the proportion difference test for independent samples based on Z-score.
+
+All tests use a 0.95 confidence level by default.
+For larger data with > 1000 observations in the reference dataset we use Jensen–Shannon divergence with a threshold = 0.1 .
+
+Requirements :
+
+Inferences
+Actuals for the inferences
+
+The target of evaluation metrics is to evaluate the quality of an machine learning model. Currently supported models are:
+
+Binary classification
+Multi-class classification
+Regression
+
+
+
+
+Metric
+Supported model
+
+
+
+
+confusion matrix binary classification & multi-class classification
+
+accuracy binary classification & multi-class classification
+
+precision binary classification & multi-class classification
+
+recall binary classification & multi-class classification
+
+f1 score binary classification & multi-class classification
+
+r_square regression
+
+mean_squared_error regression
+
+mean_absolute_error regression
+
+
+Explainability
+Requirements :
+
+The target of the explainability feature is to provide a contribution score of each feature to each individual prediction, in a try to explain how the model concluded in the specific prediction. To achieve this we define 3 Levels of confidence in the explainability feature, based on how accessible or not is the client's input:
+
+Level-0 : In this level a replacement model is trained in the same training data as client's model, and used for the explainability feature.Level-1 (pending ): In this level a surrogate model is trained in a way of trying to achieve the same predictions as client's model, and used for the explainability feature.Level-2 (pending ): Client's model is used for the explainability feature.
+Level-0 confidence
+At this level a replacement model is trained in the same training data as client's model, and used for the explainability feature. Below there is a table of used models per machine learning task.
+
+
+
+Model
+Task
+
+
+
+
+LightGBM binary classification & multi-class classification & regression
+
+
+The fine tuning of models, through a hyper-parameters exploration is a pending task for now.
+The trained model along with the inference data are used as input in LIME library, and the below report is provided - presenting the contribution of each feature in a specific prediction (the report includes all the feature contribution score to an descending order based on the absolute score value):
+{
+ "feature1" : "contribution score" ,
+ "feature2" : "contribution score" ,
+ "feature3" : "contribution score"
+}
+Level-1 confidence
+Coming soon
+Level-2 confidence
+Coming soon
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Monitors and Alerts - Whitebox
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Monitors and Alerts
+Monitors
+You can create a monitor in whitebox so that alert are created automaticaly when some value is out of bounds. Here is an example:
+from whitebox import Whitebox , MonitorStatus , MonitorMetrics , AlertSeverity
+wb = Whitebox ( host = "127.0.0.1:8000" , api_key = "some_api_key" )
+model_monitor = wb . create_model_monitor (
+model_id = "mock_model_id" ,
+name = "test" ,
+status = MonitorStatus . active ,
+metric = MonitorMetrics . accuracy ,
+severity = AlertSeverity . high ,
+email = "jackie.chan@somemail.io" ,
+lower_threshold = 0.7
+)
+Alerts
+Once the metrics reports have been produced, the monitoring alert pipeline is triggered. This means that if you have created any model monitors for a specific metric, alerts will be created if certain criteria are met, based on the thresholds and the monitor types you have specified.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Using Whitebox SDK - Whitebox
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Using Whitebox SDK
+Installation
+Installing Whitebox is a pretty easy job. Just install it like any other python package.
+Install the SDK with pip:
+
+
+
$ pip install whitebox-sdk
+
+
+
All the required packages will be automatically installed!
+Now you're good to go!
+Initial Setup
+In order to run Whitebox, you will need the application's API key.
+This key will be produced for you during the initial run of the Uvicorn live server.
+Assuming you run the server with docker compose (you can find more in the install page of the tutorial), you will see the following output:
+
+
+
$ docker compose up
+...
+<span style="color: green;">INFO</span>: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit)
+<span style="color: green;">INFO</span>: Started reloader process [4450] using StatReload
+<span style="color: green;">INFO</span>: Started server process [4452]
+<span style="color: green;">INFO</span>: Waiting for application startup.
+<span style="color: green;">INFO</span>: Created username: admin, API key: some_api_key
+<span style="color: green;">INFO</span>: Application startup complete.
+...
+
+
+
+
Info
+
Keep this API key somewhere safe!
+
If you lose it, you will need to delete the admin user in your database and re-run the live serve to produce a new key!
+
After you get the API key, all you have to do is create an instance of the Whitebox class adding your host and API key as parameters:
+from whitebox import Whitebox
+wb = Whitebox ( host = "http://127.0.0.1:8000" , api_key = "some_api_key" )
+Now you're ready to start using Whitebox!
+Models
+Creating a Model
+In order to start adding training datasets and inferences, you first need to create a model.
+Let's create a sample model:
+wb . create_model (
+name = "Model 1" ,
+type = "binary" ,
+labels = {
+'additionalProp1' : 0 ,
+'additionalProp2' : 1
+},
+prediction = "target"
+)
+For more details about the schema accepted property types visit the Models section in the SDK documentation.
+Fetching a Model
+Getting a model from the database is as easy as it sounds. You'll just need the model_id:
+wb . get_model ( "some_model_id" )
+Deleting a model
+Deleting a model is as easy as fetching a model. Just use the model_id:
+wb . delete_model ( "some_model_id" )
+
+
Warning
+
You will have to be extra careful when deleting a model because all datasets, inferences, monitors and literally everything will be deleted from the database along with the model itself!
+
Loading Training Datasets
+Once you have created a model you can start loading your data. Let's start with the training dataset!
+In our example we will create a pd.DataFrame from a .csv file. Of course you can use any method you like to create your pd.DataFrame as long as your non-processed and processed datasets have the same amount of rows (a.k.a. the same length) and there are more than one rows !
+import pandas as pd
+non_processed_df = pd . read_csv ( "path/to/file/non_processed_data.csv" )
+processed_df = pd . read_csv ( "path/to/file/processed_data.csv" )
+wb . log_training_dataset (
+model_id = "some_model_id" ,
+non_processed = non_processed_df ,
+processed = processed_df
+)
+
+
Note
+
When your training dataset is saved in the database, the model training process will begin excecuting, based on this dataset and the model it's associated with. That's why you need to load all the rows of your training dataset in the same batch.
+
Loading Inferences
+To load your inferences you have to follow the exact same procedure as with the training datasets. The only difference is that you need to provide a pd.Series with the timestamps and (optionally) a pd.Series with the actuals, whose indices should match the ones in the non-processed and processed pd.DataFrames.
+In our example let's assume that both the non-processed and processed pd.DataFrames have 10 rows each:
+import pandas as pd
+non_processed_df = pd . read_csv ( "path/to/file/non_processed_data.csv" )
+processed_df = pd . read_csv ( "path/to/file/processed_data.csv" )
+# Timestamps and actuals should have a length of 10
+timestamps = pd . Series ([ "2022-12-22T12:13:27.879738" ] * 10 )
+actuals = pd . Series ([ 0 , 1 , 1 , 1 , 0 , 0 , 1 , 1 , 0 , 0 ])
+wb . log_inferences (
+model_id = "some_model_id" ,
+non_processed = non_processed_df ,
+processed = processed_df ,
+timestamps = timestamps ,
+actuals = actuals
+)
+
+
Warning
+
Make sure you add the actuals if you already know them, because as of now, there's no ability to add them at a later time by updating the inference rows.
+
+
+
+
+
+
+
+
+
-
-
{text1}
-
type: {text2}
-
{text3}
-
-
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " id \n",
- " model_monitor_id \n",
- " timestamp \n",
- " description \n",
- " \n",
- " \n",
- " \n",
- " \n",
- " 0 \n",
- " 44 \n",
- " 1 \n",
- " 2023-03-14 \n",
- " fell below the threshold \n",
- " \n",
- " \n",
- " 1 \n",
- " 45 \n",
- " 1 \n",
- " 2023-03-16 \n",
- " fell below the threshold \n",
- " \n",
- " \n",
- "
\n",
- "
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " id \n",
- " model_monitor_id \n",
- " timestamp \n",
- " description \n",
- " \n",
- " \n",
- " \n",
- " \n",
- " 1 \n",
- " 45 \n",
- " 1 \n",
- " 2023-03-16 \n",
- " fell below the threshold \n",
- " \n",
- " \n",
- "
\n",
- "
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " id \n",
- " model_id \n",
- " name \n",
- " status \n",
- " metric \n",
- " severity \n",
- " email \n",
- " lower_threshold \n",
- " \n",
- " \n",
- " \n",
- " \n",
- " 0 \n",
- " 1 \n",
- " model_test \n",
- " my_custom_metric \n",
- " active \n",
- " accuracy \n",
- " low \n",
- " test@squaredev.io \n",
- " 0.7 \n",
- " \n",
- " \n",
- " 1 \n",
- " 2 \n",
- " model_test \n",
- " my_custom_metric \n",
- " active \n",
- " data_drift \n",
- " low \n",
- " test@squaredev.io \n",
- " 0.5 \n",
- " \n",
- " \n",
- "
\n",
- "
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " id \n",
- " model_id \n",
- " name \n",
- " status \n",
- " metric \n",
- " severity \n",
- " email \n",
- " lower_threshold \n",
- " model_monitor_id \n",
- " timestamp \n",
- " description \n",
- " \n",
- " \n",
- " \n",
- " \n",
- " 0 \n",
- " 1 \n",
- " model_test \n",
- " my_custom_metric \n",
- " active \n",
- " accuracy \n",
- " low \n",
- " test@squaredev.io \n",
- " 0.7 \n",
- " 1 \n",
- " 2023-03-16 \n",
- " fell below the threshold \n",
- " \n",
- " \n",
- "
\n",
- "
{subtitle} ",
- )
- st.plotly_chart(fig)
-
- with common_tab:
- # If dift is detected in at least one column/variable it
- # mentions drift as a whole
- drift_detected = True in drift_df.values
- subtitle = ""
-
- if drift_detected:
- subtitle = "Drift detected"
-
- fig = px.line(
- value_df,
- x="index",
- y=df_columns,
- title=f"All variables {subtitle} ",
- markers=True,
- )
- st.plotly_chart(fig)
diff --git a/whitebox/streamlit/tabs/inferences.py b/whitebox/streamlit/tabs/inferences.py
deleted file mode 100644
index 0df80ad..0000000
--- a/whitebox/streamlit/tabs/inferences.py
+++ /dev/null
@@ -1,72 +0,0 @@
-import streamlit as st
-import pandas as pd
-from utils.transformation import convert_inference_to_df
-from utils.export import structure
-import time
-
-import os, sys
-
-sys.path.insert(0, os.path.abspath("./"))
-
-from whitebox import Whitebox
-
-
-@st.cache_data(ttl=300)
-def highlight_rows(row: pd.DataFrame, pred_column: str):
- """
- Part of styling function of dataframe.
- It highlights the rows where actual is inequal to to prediction
- """
- actual = row.loc["actual"]
- pred = row.loc[pred_column]
- if actual != pred:
- color = "#8FC4C7"
- else:
- color = ""
-
- return ["background-color: {}".format(color) for r in row]
-
-
-def viz_inference_df(inf_df: pd.DataFrame, pred_column: str):
- """
- Visualises the highlighted dataframe in Streamlit
- """
- # Style and mark the columns when actual is not equal to prediction
- inf_df = inf_df.style.apply(lambda x: highlight_rows(x, pred_column), axis=1)
- st.dataframe(inf_df, width=1200, height=390)
-
-
-def create_inferences_tab(wb: Whitebox, model_id: str, pred_column: str) -> None:
- """
- Creates the Inferences tab in Streamlit.
- It visualises the dataframe of the inferences and also spawns
- the explanation part based on explainability.
- """
- with st.spinner("Loading inferences..."):
- structure()
- st.title("Inferences")
- inf = wb.get_inferences(model_id)
-
- if inf:
- inf_df = convert_inference_to_df(inf, pred_column)
-
- explain = st.checkbox("Explain inferences")
- if explain:
- col1, col2 = st.columns(2)
-
- with col1:
- # TODO: Add filter for dates (eg. Show data from 'last month')
- viz_inference_df(inf_df, pred_column)
-
- with col2:
- text_input = st.text_input(
- "Explain an inference based on id:",
- placeholder="an inference id",
- )
-
- if text_input:
- with st.spinner("Loading explanations for id: " + text_input):
- time.sleep(8)
-
- else:
- viz_inference_df(inf_df, pred_column)
diff --git a/whitebox/streamlit/tabs/monitors.py b/whitebox/streamlit/tabs/monitors.py
deleted file mode 100644
index c908da7..0000000
--- a/whitebox/streamlit/tabs/monitors.py
+++ /dev/null
@@ -1,227 +0,0 @@
-from typing import List
-import streamlit as st
-import pandas as pd
-from utils.export import structure
-from utils.transformation import (
- get_recent_alert,
- combine_monitor_with_alert_for_monitors,
-)
-from utils.load import load_config
-from utils.export import text_markdown
-
-import os, sys
-from whitebox.entities import ModelMonitor
-
-sys.path.insert(0, os.path.abspath("./"))
-
-from whitebox import Whitebox
-
-
-def add_new_monitor(wb, model_id, model_type) -> None:
- """
- Spawns the section of the addition of a new monitor
- """
- readme = load_config("config_readme.toml")
- new_monitor_name = st.text_input(
- "Monitor name",
- max_chars=30,
- help=readme["tooltips"]["monitor_name"],
- placeholder="Model name",
- )
- if new_monitor_name:
- # TODO: To include "Data Quality" in the future
- monitor_option = st.selectbox(
- "Select use case",
- ("Drift", "Model performance"),
- help=readme["tooltips"]["monitor_use_case"],
- )
- if monitor_option == "Model performance":
- if (model_type == "binary") | (model_type == "multi_class"):
- metric_option = st.selectbox(
- "Select metric",
- ("accuracy", "precision", "recall", "f1"),
- )
- else:
- metric_option = st.selectbox(
- "Select metric",
- ("r_square", "mean_squared_error", "mean_absolute_error"),
- )
- else:
- metric_option = st.selectbox(
- "Select metric",
- ("data_drift", "concept_drift"),
- )
-
- monitor_option_check = st.checkbox("Select static threshold")
-
- if monitor_option_check:
- st.write(
- text_markdown(
- readme["tooltips"]["alert_trig_monitor"], "#525462", "12px"
- )
- )
- lower_threshold = st.number_input(
- "Lower threshold",
- min_value=0.0,
- max_value=1.0,
- key="lower",
- help=readme["tooltips"]["stat_thresh_monitor"],
- )
- # TODO: upper_threshold is not currently being used
- upper_threshold = st.number_input(
- "Upper threshold",
- min_value=0.0,
- max_value=1.0,
- key="upper",
- help=readme["tooltips"]["stat_thresh_monitor"],
- )
- threshold_option_check = st.checkbox("Set actions")
-
- if threshold_option_check:
- st.write("Alert severity")
- st.write(
- text_markdown(
- readme["tooltips"]["alert_severity_monitor"], "#525462", "12px"
- )
- )
- severity = st.radio(
- "Alert severity",
- ["low", "medium", "high"],
- label_visibility="collapsed",
- )
-
- st.write("Notifications")
- st.write(
- text_markdown(
- readme["tooltips"]["notifications_monitor"], "#525462", "12px"
- )
- )
- email = st.text_input(
- "Notifications",
- placeholder="Your email...",
- label_visibility="collapsed",
- )
-
- setup_button = st.button("Complete setup")
- if setup_button:
- new_monitor = wb.create_model_monitor(
- model_id=model_id,
- name=new_monitor_name,
- status="active",
- metric=metric_option,
- severity=severity,
- email=email,
- lower_threshold=lower_threshold,
- )
- st.write(
- f"The new monitor has been created with id: ",
- new_monitor["id"],
- )
- st.write(
- "Please uncheck the 'Add new monitor' checkbox at the top to go back."
- )
-
-
-def update_model_monitor_attribute(
- wb: Whitebox, model_monitor_id: str, selected_attribute: str, value: str
-) -> bool:
- updated_model_monitor = wb.update_model_monitor(
- model_monitor_id, **{selected_attribute: value}
- )
- st.write(
- f"The status of the selected monitors has been updated to {value.capitalize()}!"
- )
-
- return updated_model_monitor
-
-
-def basic_monitor_page(
- wb: Whitebox,
- monitor_list: List[ModelMonitor],
- show_df: pd.DataFrame,
- merged_df: pd.DataFrame = None,
-) -> None:
- """
- Create the basic monitor page part.
- Displays the dataframe of the monitors and adds filters
- for activity and inactivity of found monitors.
- """
-
- st.dataframe(show_df, width=1200, height=300)
- if merged_df:
- select_box = st.selectbox(
- "Search and filter for monitors", merged_df["name"].values.tolist()
- )
- else:
- merged_df = show_df
- select_box = st.selectbox(
- "Search and filter for monitors", show_df["Name"].values.tolist()
- )
-
- if select_box:
- filtered_df = show_df[show_df["Name"].isin([select_box])]
- st.dataframe(filtered_df, width=1200, height=200)
-
- monitor = next(filter(lambda x: x["name"] == select_box, monitor_list), None)
-
- status = st.checkbox("Change the status of the selected monitors")
- delete = st.checkbox("Delete monitor")
- if status:
- status_slider = st.select_slider(
- "Select the status of the selected monitor",
- ["Active", "", "Inactive"],
- value=(""),
- )
-
- if status_slider == "Active":
- update_model_monitor_attribute(wb, monitor["id"], "status", "active")
-
- elif status_slider == "Inactive":
- update_model_monitor_attribute(wb, monitor["id"], "status", "inactive")
- if delete:
- delete_button = st.button("Delete monitor")
- if delete_button:
- deleted_model_monitor = wb.delete_model_monitor(monitor["id"])
- st.write(f"The selected monitors has been deleted!")
-
-
-def create_monitors_tab(wb: Whitebox, model_id: str, model_type: str):
- """
- Creates the monitors tabs in Streamlit
- """
- with st.spinner("Loading monitors..."):
- structure()
- st.title("Monitors")
- monitors = wb.get_model_monitors(model_id)
- alerts = wb.get_alerts(model_id)
-
- monitors_df = pd.DataFrame(monitors)
- alerts_df = pd.DataFrame(alerts)
-
- # We need to find the recent alerts for each monitor
- # then we need them to get their decriptions to show into the
- # monitors tab
- recent_alerts_df = get_recent_alert(alerts_df)
- if len(monitors_df) > 0:
- if len(recent_alerts_df) > 0:
- merged_df = combine_monitor_with_alert_for_monitors(
- monitors_df, recent_alerts_df
- )
- show_df = merged_df[["status", "name", "updated_at", "description"]]
- show_df.columns = ["Status", "Name", "Last update", "Anomaly activity"]
- else:
- show_df = monitors_df[["status", "name", "updated_at"]]
- show_df.columns = ["Status", "Name", "Last update"]
-
- add_new_monitor_check = st.checkbox(
- "Add new monitor",
- key="add",
- )
- if add_new_monitor_check:
- add_new_monitor(wb, model_id, model_type)
- else:
- if len(monitors_df) > 0:
- if len(recent_alerts_df) > 0:
- basic_monitor_page(wb, monitors, show_df, merged_df)
- else:
- basic_monitor_page(wb, monitors, show_df)
diff --git a/whitebox/streamlit/tabs/overview.py b/whitebox/streamlit/tabs/overview.py
deleted file mode 100644
index 0dff82c..0000000
--- a/whitebox/streamlit/tabs/overview.py
+++ /dev/null
@@ -1,110 +0,0 @@
-import streamlit as st
-from matplotlib import pyplot as plt
-import pandas as pd
-from numpy import ndarray
-from sklearn.metrics import ConfusionMatrixDisplay
-from cards import *
-from utils.export import structure
-from utils.load import load_config
-
-import os, sys
-
-sys.path.insert(0, os.path.abspath("./"))
-from whitebox.schemas.model import Model
-
-
-# TODO: Need to connect this one with the db.
-# Currently one shot running for training data is not supported!
-def create_classification_performance_metrics(
- base_evaluation_metrics: pd.DataFrame,
-) -> None:
- """
- Create performance metrics visualisation for classification model in Streamlit
- """
- col1, col2, col3, col4 = st.columns(4)
- col1.metric(
- label="Accuracy",
- value=base_evaluation_metrics["accuracy"].iloc[0],
- )
-
- col2.metric(
- label="Precision",
- value=base_evaluation_metrics["precision"].iloc[0],
- )
-
- col3.metric(
- label="Recall",
- value=base_evaluation_metrics["recall"].iloc[0],
- )
-
- col4.metric(
- label="F1",
- value=base_evaluation_metrics["f1"].iloc[0],
- )
-
-
-def create_regression_performance_metrics(
- base_evaluation_metrics: pd.DataFrame,
-) -> None:
- """
- Create performance metrics visualisation for regression model in Streamlit
- """
- col1, col2, col3 = st.columns(3)
- col1.metric(
- label="R2",
- value=base_evaluation_metrics["r_square"].iloc[0],
- )
-
- col2.metric(
- label="MSE",
- value=base_evaluation_metrics["mean_squared_error"].iloc[0],
- )
-
- col3.metric(
- label="MAE",
- value=base_evaluation_metrics["mean_absolute_error"].iloc[0],
- )
-
-
-def plot_confusion_matrix(confusion_matrix: ndarray, model: Model):
- st.header("Confusion matrix")
-
- if model["labels"]:
- display_labels = list(model["labels"].values())
- else:
- display_labels = None
-
- disp = ConfusionMatrixDisplay(
- confusion_matrix=confusion_matrix,
- display_labels=display_labels,
- )
- fig, ax = plt.subplots(figsize=(5, 5))
- disp.plot(ax=ax)
- st.pyplot()
-
-
-def create_overview_tab(
- model: Model, confusion_matrix: ndarray, base_evaluation_metrics: pd.DataFrame
-) -> None:
- """
- Creates the overview tab in Streamlit
- """
- with st.spinner("Loading overview of the model..."):
- readme = load_config("config_readme.toml")
- structure()
- st.title("Overview")
- st.write(card(model["name"], model["type"], model["description"]))
- st.header("Performance")
-
- with st.expander("See explanation"):
- st.write(readme["tooltips"]["overview_performance"])
-
- if model["type"] == "binary":
- create_classification_performance_metrics(base_evaluation_metrics)
- plot_confusion_matrix(confusion_matrix, model)
-
- elif model["type"] == "multi_class":
- create_classification_performance_metrics(base_evaluation_metrics)
-
- elif model["type"] == "regression":
- create_regression_performance_metrics(base_evaluation_metrics)
diff --git a/whitebox/streamlit/tabs/performance.py b/whitebox/streamlit/tabs/performance.py
deleted file mode 100644
index 7c702b4..0000000
--- a/whitebox/streamlit/tabs/performance.py
+++ /dev/null
@@ -1,60 +0,0 @@
-import streamlit as st
-import pandas as pd
-import numpy as np
-from utils.export import structure
-from utils.transformation import (
- get_dataframe_from_classification_performance_metrics,
- get_dataframe_from_regression_performance_metrics,
-)
-from utils.graphs import create_line_graph
-
-import os, sys
-
-sys.path.insert(0, os.path.abspath("./"))
-from whitebox import Whitebox
-
-
-def create_performance_graphs(performance_df: pd.DataFrame, perf_column: str) -> None:
- """
- Creates a graph based on a performance metric
- """
- viz_df = performance_df[[perf_column, "timestamp"]]
- viz_df.columns = ["score (%)", "time"]
- mean_score = round(np.mean(viz_df["score (%)"]) * 100, 2)
- subtitle = str(mean_score) + " %"
- create_line_graph(viz_df, "time", "score (%)", perf_column, subtitle, 400, 380)
-
-
-def create_performance_tab(wb: Whitebox, model_id: str, model_type: str) -> None:
- """
- Creates the performance tab in Streamlit
- """
-
- with st.spinner("Loading performance of the model..."):
- structure()
- st.title("Performance")
- performance = wb.get_performance_metrics(model_id)
-
- if performance:
- # Set the graphs in two columns (side by side)
- col1, col2 = st.columns(2)
-
- if (model_type == "binary") | (model_type == "multi_class"):
- performance_df = get_dataframe_from_classification_performance_metrics(
- performance
- )
- else:
- # For now the only case is regression
- performance_df = get_dataframe_from_regression_performance_metrics(
- performance
- )
- # Need to keep only the metrics columns to be visualised as separeted graphs
- perf_columns = performance_df.drop("timestamp", axis=1).columns
-
- for i in range(len(perf_columns)):
- if (i % 2) == 0:
- with col1:
- create_performance_graphs(performance_df, perf_columns[i])
- else:
- with col2:
- create_performance_graphs(performance_df, perf_columns[i])
diff --git a/whitebox/streamlit/tabs/sidebar.py b/whitebox/streamlit/tabs/sidebar.py
deleted file mode 100644
index af70454..0000000
--- a/whitebox/streamlit/tabs/sidebar.py
+++ /dev/null
@@ -1,228 +0,0 @@
-import streamlit as st
-from utils.load import load_config, load_image, local_css
-from utils.export import display_links, center_image
-from typing import List, Tuple, Union, Dict, Any
-from whitebox.schemas.model import Model
-from whitebox import Whitebox
-from utils.transformation import get_models_names, get_model_from_name
-
-
-@st.cache_resource
-def initialise_whitebox(host_option, api_key_option):
- wb = Whitebox(
- host=host_option,
- api_key=api_key_option,
- )
- return wb
-
-
-def create_sidebar() -> (
- Tuple[
- Union[str, None, bool],
- Union[List[str], None],
- Union[bool, None],
- Union[Whitebox, None],
- ]
-):
- """
- Creates the sidebar of a Streamlit app
-
- Args:
- model_names (List[str]): _description_
-
- Returns:
- Tuple[Union[str, None, bool], bool]: _description_
- """
- # Setup basic structure parts
- readme = load_config("config_readme.toml")
- center_image()
- st.sidebar.image(load_image("logo.png"), width=120)
- display_links(readme["links"]["repo"])
- st.sidebar.write(
- "Whitebox is an open source E2E ML monitoring platform with edge capabilities that plays nicely with kubernetes."
- )
-
- # Structure sidebar's additional expander sections
- local_css("whitebox/streamlit/utils/style.css")
- settings_expander = st.sidebar.expander("Settings", expanded=True)
-
- with settings_expander:
- host_option = st.text_input(
- "Paste here your Host URL:",
- max_chars=21,
- placeholder="http://127.0.0.1:8000",
- value="http://127.0.0.1:8000",
- help=readme["tooltips"]["host"],
- )
-
- api_key_option = st.text_input(
- "Paste here your API key:",
- max_chars=64,
- type="password",
- placeholder="your api key",
- help=readme["tooltips"]["api_key"],
- )
- setting_confirmation = st.checkbox("I'm ok, proceed with the above settings")
-
- # Initialise the other expanders only when the settings are confirmed
- if setting_confirmation:
- wb = initialise_whitebox(host_option, api_key_option)
-
- creation_model_expander = st.sidebar.expander("Create model", expanded=False)
- with creation_model_expander:
- create_new_model(wb, readme)
-
- selection_model_expander = st.sidebar.expander("Select model", expanded=False)
- with selection_model_expander:
- models_list = wb.get_models()
- model_names = get_models_names(models_list)
- model_option = st.selectbox(
- "Please select one of the found models below to be visualised:",
- model_names,
- help=readme["tooltips"]["model_option"],
- )
-
- modification_model_expander = st.sidebar.expander("Edit model", expanded=False)
- with modification_model_expander:
- st.write(f"This step will modify the selected model '{model_option}'")
- modify_selected_model(wb, models_list, readme, model_option)
-
- return model_option, models_list, setting_confirmation, wb
- else:
- return None, None, False, None
-
-
-def create_new_model(wb: Whitebox, readme: Dict[str, Any]):
- model_name = st.text_input(
- "Enter your model name:",
- max_chars=25,
- placeholder="Your model name",
- help=readme["tooltips"]["model_name"],
- )
-
- description_name = st.text_input(
- "Enter your model description:",
- max_chars=25,
- placeholder="Your model description",
- help=readme["tooltips"]["model_description"],
- )
-
- model_type_option = create_model_type_select_box(readme, "type_create")
- prediction_column_name = st.text_input(
- "Enter your target column name",
- max_chars=10,
- placeholder="Your target column name",
- help=readme["tooltips"]["target_column"],
- )
-
- granularity_amount_name = st.text_input(
- "Enter your model granularity amount",
- max_chars=5,
- placeholder="Your granularity amount",
- help=readme["tooltips"]["granularity_amount"],
- )
-
- granularity_type_option = create_model_granularity_select_box(
- readme, "granularity_create"
- )
- create_model_button = st.button("Create model")
-
- if create_model_button:
- try:
- granularity_amount = float(granularity_amount_name)
- # Have a control for geting model name, target column name and correct number in granularity amount
- if (
- (len(model_name) > 0)
- & (len(prediction_column_name) > 0)
- & (granularity_amount.is_integer())
- ):
- wb.create_model(
- name=model_name,
- description=description_name,
- type=model_type_option,
- target_column=prediction_column_name,
- granularity=granularity_amount_name + granularity_type_option,
- )
- st.write("The new model has been created!")
- except:
- pass
-
-
-def create_model_type_select_box(readme: Dict[str, Any], key: str):
- model_type_option_list = ["binary", "multi_class", "regression"]
- model_type_option = st.selectbox(
- "Please select model:",
- model_type_option_list,
- help=readme["tooltips"]["model_type"],
- key=key,
- )
- return model_type_option
-
-
-def create_model_granularity_select_box(readme: Dict[str, Any], key: str):
- model_granularity_type_option_list = ["Minutes", "Hours", "Days", "Weeks"]
- model_granularity_type_option_map = {
- "Minutes": "T",
- "Hours": "H",
- "Days": "D",
- "Weeks": "W",
- }
-
- model_granularity_type_option = st.selectbox(
- "Please select model granularity type:",
- model_granularity_type_option_list,
- help=readme["tooltips"]["granularity_type"],
- key=key,
- )
- model_granularity_type_option_mapped = (
- model_granularity_type_option_map[model_granularity_type_option] or None
- )
-
- return model_granularity_type_option_mapped
-
-
-def update_model_attribute(
- wb: Whitebox, model_id: str, selected_attribute: str, value: str
-):
- updated_model = wb.update_model(model_id, **{selected_attribute: value})
- st.write(f"Updated the selected model '{selected_attribute}'!")
-
- return updated_model
-
-
-def modify_selected_model(
- wb: Whitebox,
- models_list: Union[List[Model], None],
- readme: Dict[str, Any],
- selected_model_name: str,
-):
- # Get the id of the previous selected model
- selected_model = get_model_from_name(models_list, selected_model_name)
- if selected_model:
- model_id = selected_model["id"]
-
- model_option = st.selectbox(
- "Please select one of the below options:",
- ["delete model", "rename model", "change model description"],
- )
- if model_option == "delete model":
- delete_button = st.button("Delete model")
- if delete_button:
- delete_model = wb.delete_model(model_id)
- st.write(f"Deleted model '{selected_model}'!")
-
- elif model_option == "rename model":
- new_name_value = st.text_input("Please provide below the new model name:")
- rename_button = st.button("Rename model")
- if rename_button:
- update_model_attribute(wb, model_id, "name", new_name_value)
-
- elif model_option == "change model description":
- new_description_value = st.text_input(
- "Please provide below the new model description:"
- )
- description_button = st.button("Change model description")
- if description_button:
- update_model_attribute(
- wb, model_id, "description", new_description_value
- )
diff --git a/whitebox/streamlit/utils/export.py b/whitebox/streamlit/utils/export.py
deleted file mode 100644
index 4a01615..0000000
--- a/whitebox/streamlit/utils/export.py
+++ /dev/null
@@ -1,46 +0,0 @@
-from typing import Any, Dict, List
-import streamlit as st
-
-
-def display_links(repo_link: str) -> None:
- """Displays a repository"""
- st.sidebar.markdown(
- f"Source code ",
- unsafe_allow_html=True,
- )
-
-
-def structure() -> None:
- """Structures the tabs in Streamlit"""
- st.markdown(
- """
- {text}
',
- unsafe_allow_html=True,
- )
diff --git a/whitebox/streamlit/utils/graphs.py b/whitebox/streamlit/utils/graphs.py
deleted file mode 100644
index 2886062..0000000
--- a/whitebox/streamlit/utils/graphs.py
+++ /dev/null
@@ -1,25 +0,0 @@
-import streamlit as st
-import pandas as pd
-import plotly.express as px
-
-
-def create_line_graph(
- df: pd.DataFrame,
- x: str,
- y: str,
- title: str,
- subtitle: str,
- height: float,
- width: float,
- markers: bool = False,
-) -> None:
- """Plots a plotly chart in Streamlit"""
- fig = px.line(
- df,
- x=x,
- y=y,
- title=f"{title} {subtitle} ",
- height=height,
- width=width,
- )
- st.plotly_chart(fig)
diff --git a/whitebox/streamlit/utils/load.py b/whitebox/streamlit/utils/load.py
deleted file mode 100644
index d325baf..0000000
--- a/whitebox/streamlit/utils/load.py
+++ /dev/null
@@ -1,49 +0,0 @@
-import streamlit as st
-from typing import Tuple, Dict, Any
-import toml
-from pathlib import Path
-from PIL import Image
-
-
-@st.cache_data(ttl=300)
-def load_config(config_readme_filename: str) -> Dict[str, Any]:
- """Loads configuration files.
-
- Parameters
- ----------
- config_readme_filename : str
- Filename of readme configuration file.
-
- Returns
- -------
- dict
- Lib configuration file.
- dict
- Readme configuration file.
- """
- config_readme = toml.load(
- Path(f"whitebox/streamlit/config/{config_readme_filename}")
- )
- return dict(config_readme)
-
-
-@st.cache_data(ttl=300)
-def load_image(image_name: str):
- """Displays an image.
-
- Parameters
- ----------
- image_name : str
- Local path of the image.
-
- Returns
- -------
- Image
- Image to be displayed.
- """
- return Image.open(f"whitebox/streamlit/references/{image_name}")
-
-
-def local_css(file_name: str):
- with open(file_name) as f:
- st.markdown(f"", unsafe_allow_html=True)
diff --git a/whitebox/streamlit/utils/style.css b/whitebox/streamlit/utils/style.css
deleted file mode 100644
index daa9126..0000000
--- a/whitebox/streamlit/utils/style.css
+++ /dev/null
@@ -1,3 +0,0 @@
-div[data-testid="stExpander"] div[role="button"] p {
- font-size: 1.5rem;
-}
\ No newline at end of file
diff --git a/whitebox/streamlit/utils/transformation.py b/whitebox/streamlit/utils/transformation.py
deleted file mode 100644
index 0cfda7c..0000000
--- a/whitebox/streamlit/utils/transformation.py
+++ /dev/null
@@ -1,209 +0,0 @@
-import pandas as pd
-from pandas.api.types import is_datetime64_any_dtype
-from typing import List, Tuple, Union
-import streamlit as st
-import os, sys
-
-sys.path.insert(0, os.path.abspath("./"))
-
-from whitebox.schemas.driftingMetric import DriftingMetricBase
-from whitebox.schemas.performanceMetric import (
- RegressionMetrics,
- BinaryClassificationMetrics,
- MultiClassificationMetrics,
-)
-from whitebox.schemas.model import Model
-from whitebox.schemas.inferenceRow import InferenceRow
-
-
-@st.cache_data(ttl=300)
-def get_models_names(models_list: List[Model]) -> Union[List[str], None]:
- """Gets a Model list and returns a list with the models' names"""
- if len(models_list) > 0:
- models_df = pd.DataFrame(models_list)
- return models_df["name"].to_list()
- else:
- return models_list
-
-
-@st.cache_data(ttl=300)
-def get_model_from_name(
- models_list: Union[List[Model], None], model_name: str
-) -> Union[Model, None]:
- """
- Gets a model name and a models List and returns the
- entire model object with the corresponding name
- """
- if len(models_list) > 0:
- models_df = pd.DataFrame(models_list)
- model_df = models_df.loc[models_df["name"] == model_name].reset_index(drop=True)
- return model_df.to_dict(orient="records")[0]
-
-
-@st.cache_data(ttl=300)
-def convert_drift_timeseries_dict_to_pd(timeseries_dict) -> pd.DataFrame:
- """Converts the drift timeseries dict to a pandas dataframe object"""
- df = pd.DataFrame.from_dict(timeseries_dict, orient="index")
- df = df.reset_index()
- df["index"] = pd.to_datetime(df["index"])
-
- return df
-
-
-@st.cache_data(ttl=300)
-def export_drift_timeseries(
- drift: List[DriftingMetricBase],
-) -> Tuple[pd.DataFrame, pd.DataFrame]:
- """Exports the drift object to 2 pandas dataframe objects as timeseries"""
- timeseries_value = {}
- timeseries_drift = {}
-
- for i in range(len(drift)):
- drift_by_columns = drift[i]["data_drift_summary"]["drift_by_columns"]
- temp_col_value_dict = {}
- temp_col_drift_dict = {}
- for j in drift_by_columns.keys():
- temp_col_value_dict[j] = drift_by_columns[j]["drift_score"]
- temp_col_drift_dict[j] = drift_by_columns[j]["drift_detected"]
-
- timeseries_value[drift[i]["timestamp"]] = temp_col_value_dict
- timeseries_drift[drift[i]["timestamp"]] = temp_col_drift_dict
-
- value_df = convert_drift_timeseries_dict_to_pd(timeseries_value)
- drift_df = convert_drift_timeseries_dict_to_pd(timeseries_drift)
-
- return value_df, drift_df
-
-
-@st.cache_data(ttl=300)
-def get_dataframe_from_regression_performance_metrics(
- performance: List[RegressionMetrics],
-) -> pd.DataFrame:
- df = pd.DataFrame(performance)
- df = df[["r_square", "mean_squared_error", "mean_absolute_error", "timestamp"]]
-
- return df
-
-
-@st.cache_data(ttl=300)
-def get_dataframe_from_classification_performance_metrics(
- performance: Union[
- List[BinaryClassificationMetrics], List[MultiClassificationMetrics], None
- ]
-) -> Union[pd.DataFrame, None]:
- """
- Gets pandas dataframe timeseries out of classification
- performance metrics
- """
- if performance:
- timeseries = {}
- for i in range(len(performance)):
- acc = performance[i]["accuracy"]
- prec = performance[i]["precision"]["macro"]
- rec = performance[i]["recall"]["macro"]
- f1 = performance[i]["f1"]["macro"]
- timeseries[performance[i]["timestamp"]] = {
- "accuracy": acc,
- "precision": prec,
- "recall": rec,
- "f1": f1,
- }
-
- performance_df = pd.DataFrame.from_dict(
- timeseries, orient="index"
- ).reset_index()
- performance_df["index"] = pd.to_datetime(performance_df["index"])
-
- performance_df = performance_df.rename({"index": "timestamp"}, axis=1)
- return performance_df
-
-
-@st.cache_data(ttl=300)
-def adjust_inference_column_positions(
- inf_df: pd.DataFrame, target_column: str
-) -> pd.DataFrame:
- """Adjust a dataframe's columns positions"""
- df_columns = inf_df.columns
- # Find only the feature columns
- df_feature_columns = df_columns.drop([target_column, "timestamp", "id", "actual"])
- # Inser the id and timestamp in front positions
- df_adj_columns = df_feature_columns.insert(0, "id")
- df_adj_columns = df_adj_columns.insert(1, "timestamp")
- # Append target column and actual column
- df_adj_columns = df_adj_columns.to_list()
- df_adj_columns.append(target_column)
- df_adj_columns.append("actual")
-
- return inf_df.reindex(columns=df_adj_columns)
-
-
-@st.cache_data(ttl=300)
-def convert_inference_to_df(
- inf_list: List[InferenceRow], target_column: str
-) -> pd.DataFrame:
- """Gets pandas dataframe timeseries out of given inference rows"""
- temp_full_dict = {}
- for i in range(len(inf_list)):
- temp_row_dict = inf_list[i]["nonprocessed"]
- temp_row_dict["timestamp"] = inf_list[i]["timestamp"]
- temp_row_dict["id"] = inf_list[i]["id"]
- temp_row_dict["actual"] = inf_list[i]["actual"]
-
- temp_full_dict[i] = temp_row_dict
-
- inf_df = pd.DataFrame.from_dict(temp_full_dict, orient="index")
- return adjust_inference_column_positions(inf_df, target_column)
-
-
-@st.cache_data(ttl=300)
-def get_recent_alert(alerts_df: pd.DataFrame) -> pd.DataFrame:
- """
- Function that gets alerts dataframe and results on the
- most recent alert row for each unique id
- """
- if len(alerts_df) > 0:
- alerts_df["timestamp"] = pd.to_datetime(alerts_df["timestamp"])
-
- # datetime with no timezone
- if is_datetime64_any_dtype(alerts_df["timestamp"]):
- alerts_df["timestamp"] = alerts_df["timestamp"].dt.tz_localize(None)
-
- # sort the dataframe by 'date' column in descending order
- alerts_df = alerts_df.sort_values(by="timestamp", ascending=False)
- # drop duplicates based on 'id' column, keeping only the first occurrence (most recent date)
- alerts_df = alerts_df.drop_duplicates(subset="model_monitor_id", keep="first")
-
- return alerts_df.reset_index(drop=True)
- else:
- return alerts_df
-
-
-@st.cache_data(ttl=300)
-def combine_monitor_with_alert_for_alerts(
- monitor_df: pd.DataFrame, alert_df: pd.DataFrame
-) -> pd.DataFrame:
- """Merges two df together based on specific keys and columns for alerts part"""
- merged_df = pd.merge(
- monitor_df[["id", "metric", "name"]],
- alert_df[["model_monitor_id", "timestamp", "description"]],
- how="right",
- left_on="id",
- right_on="model_monitor_id",
- )
- return merged_df
-
-
-@st.cache_data(ttl=300)
-def combine_monitor_with_alert_for_monitors(
- monitor_df: pd.DataFrame, alert_df: pd.DataFrame
-) -> pd.DataFrame:
- """Merges two df together based on specific keys and columns for monitors part"""
-
- merged_df = pd.merge(
- monitor_df,
- alert_df[["model_monitor_id", "description"]],
- how="left",
- left_on="id",
- right_on="model_monitor_id",
- )
- return merged_df
diff --git a/whitebox/tests/__init__.py b/whitebox/tests/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/whitebox/tests/unit_tests/test_unit.py b/whitebox/tests/unit_tests/test_unit.py
deleted file mode 100644
index 63ba3e7..0000000
--- a/whitebox/tests/unit_tests/test_unit.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from whitebox.cron_tasks.shared import change_timestamp
-from datetime import datetime
-
-
-class TestNodes:
- def test_round_timestamp(self):
- timestamp = datetime(2023, 3, 7, 15, 34, 23)
- start_time = datetime(2023, 3, 6)
-
- assert change_timestamp(timestamp, start_time, 15, "T") == datetime(
- 2023, 3, 7, 15, 45
- )
- assert change_timestamp(timestamp, start_time, 5, "H") == datetime(
- 2023, 3, 7, 16, 0
- )
- assert change_timestamp(timestamp, start_time, 2, "D") == datetime(2023, 3, 8)
- assert change_timestamp(timestamp, start_time, 1, "W") == datetime(2023, 3, 13)
diff --git a/whitebox/tests/utils/maps.py b/whitebox/tests/utils/maps.py
deleted file mode 100644
index fbf8933..0000000
--- a/whitebox/tests/utils/maps.py
+++ /dev/null
@@ -1,47 +0,0 @@
-v1_test_order_map = [
- "health",
- "models_no_api_key",
- "models_wrong_api_key",
- "cron_tasks_run_no_models",
- "models_create",
- "models_get_all",
- "models_get",
- "models_update",
- "cron_tasks_run_no_inference",
- "inference_rows_create",
- "inference_rows_create_many",
- "inference_rows_get_model's_all",
- "inference_rows_get",
- "dataset_rows_wrong_training_dataset",
- "dataset_rows_create",
- "dataset_rows_get_model's_all",
- "inference_rows_xai",
- "model_monitor_create",
- "model_monitors_get_model_all",
- "model_monitor_update",
- "cron_tasks_run_ok",
- "performance_metrics_get_model_all",
- "drifting_metrics_get_model_all",
- "model_integrity_metrics_get_model_all",
- "alerts_get",
- "inference_rows_create_many_after_x_time",
- "cron_tasks_run_after_x_time",
- "drifting_metrics_get_binary_model_after_x_time",
- "model_monitor_delete",
- "models_delete",
- # SDK tests
- "sdk_init",
- "sdk_create_model",
- "sdk_get_model",
- "sdk_delete_model",
- "sdk_log_training_dataset",
- "sdk_log_inferences",
- "sdk_create_model_monitor",
- "sdk_update_model_monitor",
- "sdk_delete_model_monitor",
- "sdk_get_alerts",
- "sdk_get_drifting_metrics",
- "sdk_get_descriptive_statistics",
- "sdk_get_performance_metrics",
- "sdk_get_xai_row",
-]
diff --git a/whitebox/tests/v1/__init__.py b/whitebox/tests/v1/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/whitebox/tests/v1/conftest.py b/whitebox/tests/v1/conftest.py
deleted file mode 100644
index ea3280f..0000000
--- a/whitebox/tests/v1/conftest.py
+++ /dev/null
@@ -1,72 +0,0 @@
-import shutil
-from fastapi.testclient import TestClient
-from pytest import fixture
-from sqlalchemy.orm import close_all_sessions
-from whitebox import crud
-from whitebox.core.settings import get_settings
-from whitebox.entities.Base import Base
-from whitebox.main import app
-from whitebox.sdk.whitebox import Whitebox
-from whitebox.tests.utils.maps import v1_test_order_map
-from whitebox.utils.passwords import decrypt_api_key
-from whitebox.core.db import SessionLocal, engine
-
-settings = get_settings()
-
-
-def get_order_number(task):
- return v1_test_order_map.index(task)
-
-
-@fixture(scope="session")
-def client():
- with TestClient(app) as client:
- yield client
-
-
-@fixture(scope="session")
-def api_key():
- db = SessionLocal()
-
- user = crud.users.get_first_by_filter(db=db, username="admin")
- api_key = (
- decrypt_api_key(user.api_key, settings.SECRET_KEY.encode())
- if settings.SECRET_KEY
- else user.api_key
- )
-
- yield api_key
-
-
-@fixture(scope="session", autouse=True)
-def drop_db():
- yield
- # Removes the folder "test_model" with the models trained during testing
- test_model_path = settings.MODEL_PATH
- shutil.rmtree(test_model_path)
-
- close_all_sessions()
- # Drops all test database tables
- Base.metadata.drop_all(engine)
-
-
-class TestsState:
- user: dict = {}
- model_binary: dict = {}
- model_multi: dict = {}
- model_multi_2: dict = {}
- model_multi_3: dict = {}
- model_regression: dict = {}
- inference_row_multi: dict = {}
- inference_row_binary: dict = {}
- concept_drift_monitor: dict = {}
-
-
-state = TestsState()
-
-
-class TestsSDKState:
- wb: Whitebox
-
-
-state_sdk = TestsSDKState()
diff --git a/whitebox/tests/v1/mock_data.py b/whitebox/tests/v1/mock_data.py
deleted file mode 100644
index 4935b1c..0000000
--- a/whitebox/tests/v1/mock_data.py
+++ /dev/null
@@ -1,602 +0,0 @@
-from datetime import datetime
-from whitebox.schemas.model import ModelType
-from sklearn.datasets import load_breast_cancer, load_wine, load_diabetes
-import pandas as pd
-from copy import deepcopy
-
-from whitebox.schemas.modelMonitor import AlertSeverity, MonitorMetrics, MonitorStatus
-
-user_create_payload = dict(username="admin")
-
-model_binary_create_payload = dict(
- name="Model Binary",
- description="Model description",
- type=ModelType.binary,
- labels={"label_1": 0, "label_2": 1},
- target_column="target",
- granularity="1D",
-)
-
-model_multi_create_payload = dict(
- name="Model 1 Multi",
- description="Model 1 description",
- type=ModelType.multi_class,
- labels={"label_1": 0, "label_2": 1, "label_3": 2},
- target_column="target",
- granularity="15T",
-)
-
-model_multi_2_create_payload = dict(
- name="Model 2 Multi",
- description="Model 2 description",
- type=ModelType.multi_class,
- labels={"label_1": 0, "label_2": 1, "label_3": 2},
- target_column="target",
- granularity="2H",
-)
-
-model_multi_3_create_payload = dict(
- name="Model 3",
- description="Model 3 description",
- type=ModelType.multi_class,
- labels={"label_1": 0, "label_2": 1, "label_3": 2},
- target_column="target",
- granularity="1W",
-)
-
-model_regression_create_payload = dict(
- name="Regression Model",
- description="Regression Model description",
- type=ModelType.regression,
- target_column="target",
- granularity="1D",
-)
-
-model_update_payload = dict(
- name="Model 2 - categorical",
- description="Model 2 description updated",
-)
-
-# dataset rows data for both binary and multiclass models
-dataset_rows_single_row_column_payload = [
- {
- "nonprocessed": {},
- "processed": {"additionalProp1": 0, "additionalProp2": 0, "target": 1},
- }
-]
-
-dataset_rows_no_prediction_column_payload = [
- {
- "nonprocessed": {},
- "processed": {"additionalProp1": 0, "additionalProp2": 0},
- },
- {
- "nonprocessed": {},
- "processed": {"additionalProp1": 1, "additionalProp2": 0},
- },
-]
-
-dataset_rows_one_prediction_value_payload = [
- {
- "nonprocessed": {},
- "processed": {"additionalProp1": 0, "additionalProp2": 0, "target": 0},
- },
- {
- "nonprocessed": {},
- "processed": {"additionalProp1": 1, "additionalProp2": 0, "target": 0},
- },
-]
-
-
-df_load_multi = load_wine()
-df_multi = pd.DataFrame(df_load_multi.data, columns=df_load_multi.feature_names)
-df_multi["target"] = df_load_multi.target
-df_multi = df_multi.tail(100)
-
-dict_multi_data = df_multi.to_dict(orient="records")
-dataset_rows_create_multi_class_payload = [
- {"processed": x, "nonprocessed": x} for x in dict_multi_data
-]
-
-df_load_binary = load_breast_cancer()
-df_binary = pd.DataFrame(df_load_binary.data, columns=df_load_binary.feature_names)
-df_binary["target"] = df_load_binary.target
-df_binary = df_binary.tail(100)
-
-dict_binary_data = df_binary.to_dict(orient="records")
-dataset_rows_create_binary_payload = [
- {"processed": x, "nonprocessed": x} for x in dict_binary_data
-]
-
-df_load_reg = load_diabetes()
-df_reg = pd.DataFrame(df_load_reg.data, columns=df_load_reg.feature_names)
-df_reg["target"] = df_load_reg.target
-df_reg = df_reg.tail(100)
-
-dict_reg_data = df_reg.to_dict(orient="records")
-dataset_rows_create_reg_payload = [
- {"processed": x, "nonprocessed": x} for x in dict_reg_data
-]
-
-dataset_rows_create_wrong_model_payload = list(
- (
- dict(
- model_id="wrong_model_id",
- nonprocessed={"sex": "male"},
- processed={"sex": 0},
- ),
- dict(
- model_id="wrong_model_id",
- nonprocessed={"sex": "female"},
- processed={"sex": 1},
- ),
- )
-)
-
-
-# inference rows data for both binary and multiclaas models
-df_multi_inference = df_multi.tail(10)
-dict_multi_inferences = df_multi_inference.to_dict(orient="records")
-multi_actuals = [0, 2, 0, 1, 2, 1, 1, 2, 0, 1]
-inference_row_create_many_multi_payload = [
- {
- "timestamp": str(datetime(2023, 3, 6, 12, 13, 25)),
- "processed": x,
- "nonprocessed": x,
- "actual": multi_actuals[i],
- }
- for i, x in enumerate(dict_multi_inferences)
-]
-
-inference_row_create_single_row_payload = inference_row_create_many_multi_payload[0]
-inference_row_create_many_multi_no_actual_payload = deepcopy(
- inference_row_create_many_multi_payload
-)
-for x in inference_row_create_many_multi_no_actual_payload:
- del x["actual"]
-
-inference_row_create_many_multi_mixed_actuals_payload = (
- inference_row_create_many_multi_no_actual_payload
- + inference_row_create_many_multi_payload
-)
-
-# This is the body of the request coming from the sdk
-df_binary_inference = df_binary.tail(10)
-dict_binary_inferences = df_binary_inference.to_dict(orient="records")
-binary_actuals = [0, 1, 1, 0, 1, 0, 1, 1, 1, 0]
-inference_row_create_many_binary_payload = [
- {
- "timestamp": str(datetime(2023, 3, 6, 12, 13, 25)),
- "processed": x,
- "nonprocessed": x,
- "actual": binary_actuals[i],
- }
- for i, x in enumerate(dict_binary_inferences)
-]
-
-df_reg_inference = df_reg.tail(10)
-dict_reg_inferences = df_reg_inference.to_dict(orient="records")
-reg_actuals = [103, 20, 50, 64, 48, 198, 105, 138, 250, 57]
-inference_row_create_many_reg_payload = [
- {
- "timestamp": str(datetime(2023, 3, 6, 12, 13, 25)),
- "processed": x,
- "nonprocessed": x,
- "actual": reg_actuals[i],
- }
- for i, x in enumerate(dict_reg_inferences)
-]
-
-timestamps = pd.Series(["2022-12-22T12:13:27.879738"] * 10)
-mixed_actuals = pd.Series([0, 1, None, 1, 0, None, None, 1, 0, None])
-
-model_monitor_accuracy_create_payload = dict(
- name="accuracy monitor",
- status=MonitorStatus.active,
- metric=MonitorMetrics.accuracy,
- lower_threshold=0.85,
- severity=AlertSeverity.low,
- email="example@whitebox.io",
-)
-
-model_monitor_f1_create_payload = dict(
- name="f1 Monitor",
- status=MonitorStatus.active,
- metric=MonitorMetrics.f1,
- lower_threshold=0.85,
- severity=AlertSeverity.low,
- email="example@whitebox.io",
-)
-
-model_monitor_data_drift_create_payload = dict(
- name="data drift monitor",
- status=MonitorStatus.active,
- metric=MonitorMetrics.data_drift,
- feature="concavity error",
- severity=AlertSeverity.low,
- email="example@whitebox.io",
-)
-
-model_monitor_concept_drift_create_payload = dict(
- name="concept drift monitor",
- status=MonitorStatus.active,
- metric=MonitorMetrics.concept_drift,
- feature="concavity error",
- severity=AlertSeverity.low,
- email="example@whitebox.io",
-)
-
-model_monitor_precision_create_payload = dict(
- name="precision monitor",
- status=MonitorStatus.active,
- metric=MonitorMetrics.precision,
- lower_threshold=0.85,
- severity=AlertSeverity.low,
- email="example@whitebox.io",
-)
-
-model_monitor_r_square_create_payload = dict(
- name="r_square monitor",
- status=MonitorStatus.active,
- metric=MonitorMetrics.r_square,
- lower_threshold=0.85,
- severity=AlertSeverity.low,
- email="example@whitebox.io",
-)
-
-model_monitor_no_threshold_create_payload = dict(
- name="no threshold monitor",
- status=MonitorStatus.active,
- metric=MonitorMetrics.r_square,
- severity=AlertSeverity.low,
- email="example@whitebox.io",
-)
-
-model_monitor_no_feature_create_payload = dict(
- name="no feature monitor",
- status=MonitorStatus.active,
- metric=MonitorMetrics.data_drift,
- severity=AlertSeverity.low,
- email="example@whitebox.io",
-)
-
-model_monitor_feature_same_as_target_create_payload = dict(
- name="feature same as target monitor",
- status=MonitorStatus.active,
- metric=MonitorMetrics.data_drift,
- feature="target",
- severity=AlertSeverity.low,
- email="example@whitebox.io",
-)
-
-model_monitor_feature_not_in_columns_create_payload = dict(
- name="feature not in columns monitor",
- status=MonitorStatus.active,
- metric=MonitorMetrics.data_drift,
- feature="not_a_column",
- severity=AlertSeverity.low,
- email="example@whitebox.io",
-)
-
-
-model_monitor_update_payload = dict(
- name="concept drift monitor updated", lower_threshold=0.54
-)
-
-alert_payload = {
- "id": "c2a25e32-4e40-40da-a83f-fc299514866e",
- "created_at": "2023-02-09T12:52:50.547557",
- "updated_at": "2023-02-09T12:52:50.547557",
- "model_id": "11737584-a356-4de5-924c-9894dfcd0e0d",
- "model_monitor_id": "fa3a12f3-6801-4c25-8154-8b627e1d37b1",
- "timestamp": "2023-02-09T12:52:50.547525",
- "description": 'Data drift found in "concavity error" feature.',
-}
-
-performance_metrics_report_payload = [
- {
- "id": "f7030044-e0c2-4493-8aea-f66e6564efb7",
- "created_at": "2023-02-06T09:53:00.343616",
- "updated_at": "2023-02-06T09:53:00.343616",
- "model_id": "dbb7f384-cf46-4884-bca3-7787e049a3d5",
- "timestamp": "2023-02-06T09:53:00.343548",
- "accuracy": 0.6,
- "precision": 0.25,
- "recall": 0.5,
- "f1": 0.3333333333333333,
- "true_negative": 5,
- "false_positive": 3,
- "false_negative": 1,
- "true_positive": 1,
- }
-]
-
-descriptive_statistics_report_payload = [
- {
- "id": "d4f18837-65f6-45c2-af7c-34dcdc4aabf9",
- "created_at": "2023-02-06T09:53:00.370639",
- "updated_at": "2023-02-06T09:53:00.370639",
- "model_id": "bc01a212-b01e-48b4-aeb5-4a0a3cfa3c98",
- "timestamp": "2023-02-06T09:53:00.370617",
- "feature_metrics": {
- "missing_count": {
- "alcohol": 0,
- "malic_acid": 0,
- "ash": 0,
- "alcalinity_of_ash": 0,
- "magnesium": 0,
- "total_phenols": 0,
- "flavanoids": 0,
- "nonflavanoid_phenols": 0,
- "proanthocyanins": 0,
- "color_intensity": 0,
- "hue": 0,
- "od280/od315_of_diluted_wines": 0,
- "proline": 0,
- "target": 0,
- },
- "non_missing_count": {
- "alcohol": 10,
- "malic_acid": 10,
- "ash": 10,
- "alcalinity_of_ash": 10,
- "magnesium": 10,
- "total_phenols": 10,
- "flavanoids": 10,
- "nonflavanoid_phenols": 10,
- "proanthocyanins": 10,
- "color_intensity": 10,
- "hue": 10,
- "od280/od315_of_diluted_wines": 10,
- "proline": 10,
- "target": 10,
- },
- "mean": {
- "alcohol": 13.379000000000001,
- "malic_acid": 3.564,
- "ash": 2.493,
- "alcalinity_of_ash": 21.6,
- "magnesium": 102.3,
- "total_phenols": 1.6620000000000001,
- "flavanoids": 0.699,
- "nonflavanoid_phenols": 0.44499999999999995,
- "proanthocyanins": 1.1889999999999998,
- "color_intensity": 8.595999899999999,
- "hue": 0.6399999999999999,
- "od280/od315_of_diluted_wines": 1.6970000000000003,
- "proline": 674.5,
- "target": 2.0,
- },
- "minimum": {
- "alcohol": 12.2,
- "malic_acid": 2.39,
- "ash": 2.26,
- "alcalinity_of_ash": 19.0,
- "magnesium": 86.0,
- "total_phenols": 1.25,
- "flavanoids": 0.49,
- "nonflavanoid_phenols": 0.27,
- "proanthocyanins": 0.64,
- "color_intensity": 5.5,
- "hue": 0.57,
- "od280/od315_of_diluted_wines": 1.56,
- "proline": 470.0,
- "target": 2.0,
- },
- "maximum": {
- "alcohol": 14.16,
- "malic_acid": 5.65,
- "ash": 2.86,
- "alcalinity_of_ash": 25.0,
- "magnesium": 120.0,
- "total_phenols": 2.05,
- "flavanoids": 0.96,
- "nonflavanoid_phenols": 0.56,
- "proanthocyanins": 1.54,
- "color_intensity": 10.2,
- "hue": 0.74,
- "od280/od315_of_diluted_wines": 1.92,
- "proline": 840.0,
- "target": 2.0,
- },
- "sum": {
- "alcohol": 133.79000000000002,
- "malic_acid": 35.64,
- "ash": 24.93,
- "alcalinity_of_ash": 216.0,
- "magnesium": 1023.0,
- "total_phenols": 16.62,
- "flavanoids": 6.989999999999999,
- "nonflavanoid_phenols": 4.449999999999999,
- "proanthocyanins": 11.889999999999999,
- "color_intensity": 85.959999,
- "hue": 6.3999999999999995,
- "od280/od315_of_diluted_wines": 16.970000000000002,
- "proline": 6745.0,
- "target": 20.0,
- },
- "standard_deviation": {
- "alcohol": 0.5907894906159238,
- "malic_acid": 1.1073311258256142,
- "ash": 0.2058613341278272,
- "alcalinity_of_ash": 2.3664319132398464,
- "magnesium": 11.80442478244681,
- "total_phenols": 0.24334703157790474,
- "flavanoids": 0.14224001624796806,
- "nonflavanoid_phenols": 0.08396427811873335,
- "proanthocyanins": 0.3045743842734572,
- "color_intensity": 1.4311393049673125,
- "hue": 0.05291502622129182,
- "od280/od315_of_diluted_wines": 0.12356284950492914,
- "proline": 130.39363481397396,
- "target": 0.0,
- },
- "variance": {
- "alcohol": 0.34903222222222274,
- "malic_acid": 1.2261822222222223,
- "ash": 0.04237888888888891,
- "alcalinity_of_ash": 5.599999999999999,
- "magnesium": 139.34444444444443,
- "total_phenols": 0.059217777777777765,
- "flavanoids": 0.02023222222222222,
- "nonflavanoid_phenols": 0.0070500000000000024,
- "proanthocyanins": 0.09276555555555556,
- "color_intensity": 2.048159710222322,
- "hue": 0.002800000000000001,
- "od280/od315_of_diluted_wines": 0.015267777777777769,
- "proline": 17002.5,
- "target": 0.0,
- },
- },
- }
-]
-
-drifting_metrics_report_payload = [
- {
- "id": "d11f2313-12e3-4533-b16d-7689175ad617",
- "created_at": "2023-02-06T09:53:00.450466",
- "updated_at": "2023-02-06T09:53:00.450466",
- "model_id": "e8e5ff00-0212-4a1e-bcdb-0ef9183c867f",
- "timestamp": "2023-02-06T09:53:00.450435",
- "concept_drift_summary": {
- "concept_drift_summary": {
- "column_name": "target",
- "column_type": "cat",
- "stattest_name": "Z-test p_value",
- "drift_score": 0.0010364142456651404,
- "drift_detected": True,
- "stattest_threshold": 0.05,
- },
- "column_correlation": {
- "column_name": "target",
- "current": {},
- "reference": {},
- },
- },
- "data_drift_summary": {
- "number_of_columns": 13,
- "number_of_drifted_columns": 7,
- "share_of_drifted_columns": 0.5384615384615384,
- "dataset_drift": True,
- "drift_by_columns": {
- "alcalinity_of_ash": {
- "column_name": "alcalinity_of_ash",
- "column_type": "num",
- "stattest_name": "K-S p_value",
- "drift_score": 0.5163177458201748,
- "drift_detected": False,
- "threshold": 0.05,
- },
- "alcohol": {
- "column_name": "alcohol",
- "column_type": "num",
- "stattest_name": "K-S p_value",
- "drift_score": 0.0012612112534500996,
- "drift_detected": True,
- "threshold": 0.05,
- },
- "ash": {
- "column_name": "ash",
- "column_type": "num",
- "stattest_name": "K-S p_value",
- "drift_score": 0.23438597334285016,
- "drift_detected": False,
- "threshold": 0.05,
- },
- "color_intensity": {
- "column_name": "color_intensity",
- "column_type": "num",
- "stattest_name": "K-S p_value",
- "drift_score": 0.00015242693342585546,
- "drift_detected": True,
- "threshold": 0.05,
- },
- "flavanoids": {
- "column_name": "flavanoids",
- "column_type": "num",
- "stattest_name": "K-S p_value",
- "drift_score": 0.0003384276858653337,
- "drift_detected": True,
- "threshold": 0.05,
- },
- "hue": {
- "column_name": "hue",
- "column_type": "num",
- "stattest_name": "K-S p_value",
- "drift_score": 0.0002500181491633883,
- "drift_detected": True,
- "threshold": 0.05,
- },
- "magnesium": {
- "column_name": "magnesium",
- "column_type": "num",
- "stattest_name": "K-S p_value",
- "drift_score": 0.08450859981880006,
- "drift_detected": False,
- "threshold": 0.05,
- },
- "malic_acid": {
- "column_name": "malic_acid",
- "column_type": "num",
- "stattest_name": "K-S p_value",
- "drift_score": 0.033698127881912385,
- "drift_detected": True,
- "threshold": 0.05,
- },
- "nonflavanoid_phenols": {
- "column_name": "nonflavanoid_phenols",
- "column_type": "num",
- "stattest_name": "K-S p_value",
- "drift_score": 0.30861039864383094,
- "drift_detected": False,
- "threshold": 0.05,
- },
- "od280/od315_of_diluted_wines": {
- "column_name": "od280/od315_of_diluted_wines",
- "column_type": "num",
- "stattest_name": "K-S p_value",
- "drift_score": 0.0006076739224878145,
- "drift_detected": True,
- "threshold": 0.05,
- },
- "proanthocyanins": {
- "column_name": "proanthocyanins",
- "column_type": "num",
- "stattest_name": "K-S p_value",
- "drift_score": 0.09673060972050829,
- "drift_detected": False,
- "threshold": 0.05,
- },
- "proline": {
- "column_name": "proline",
- "column_type": "num",
- "stattest_name": "K-S p_value",
- "drift_score": 0.0328848734766648,
- "drift_detected": True,
- "threshold": 0.05,
- },
- "total_phenols": {
- "column_name": "total_phenols",
- "column_type": "num",
- "stattest_name": "K-S p_value",
- "drift_score": 0.082145120769578,
- "drift_detected": False,
- "threshold": 0.05,
- },
- },
- },
- }
-]
-
-inference_row_xai_payload = {
- "mean perimeter": -0.23641989216827916,
- "mean concavity": -0.16854638344893477,
- "worst radius": -0.16657884627852115,
- "mean concave points": -0.1260514980862101,
- "worst perimeter": -0.08533726601274261,
- "worst texture": -0.07905090544100012,
- "mean texture": -0.05969642438812264,
- "worst concave points": -0.03310756238929911,
- "mean radius": -0.03244286722644983,
- "worst symmetry": 0.02852296352595751,
-}
diff --git a/whitebox/tests/v1/test_alerts.py b/whitebox/tests/v1/test_alerts.py
deleted file mode 100644
index 64ba6b7..0000000
--- a/whitebox/tests/v1/test_alerts.py
+++ /dev/null
@@ -1,31 +0,0 @@
-import pytest
-from whitebox import schemas
-from whitebox.tests.v1.conftest import get_order_number, state
-from fastapi import status
-
-
-@pytest.mark.order(get_order_number("alerts_get"))
-def test_alerts_get_model_all(client, api_key):
- response_all = client.get(
- f"/v1/alerts",
- headers={"api-key": api_key},
- )
-
- response_model_all = client.get(
- f"/v1/alerts?model_id={state.model_multi['id']}",
- headers={"api-key": api_key},
- )
-
- response_wrong_model = client.get(
- f"/v1/alerts?model_id=wrong_model_id",
- headers={"api-key": api_key},
- )
-
- assert len(response_all.json()) == 6
-
- assert response_all.status_code == status.HTTP_200_OK
- assert response_model_all.status_code == status.HTTP_200_OK
- assert response_wrong_model.status_code == status.HTTP_404_NOT_FOUND
-
- validated = [schemas.Alert(**m) for m in response_all.json()]
- validated = [schemas.Alert(**m) for m in response_model_all.json()]
diff --git a/whitebox/tests/v1/test_cron_tasks.py b/whitebox/tests/v1/test_cron_tasks.py
deleted file mode 100644
index 27cdc62..0000000
--- a/whitebox/tests/v1/test_cron_tasks.py
+++ /dev/null
@@ -1,27 +0,0 @@
-import pytest
-from whitebox.tests.v1.conftest import get_order_number
-from fastapi import status
-
-
-@pytest.mark.order(get_order_number("cron_tasks_run_no_models"))
-def test_cron_tasks_no_models(client):
- response = client.post("/v1/cron-tasks/run")
- assert response.status_code == status.HTTP_200_OK
-
-
-@pytest.mark.order(get_order_number("cron_tasks_run_no_inference"))
-def test_cron_tasks_no_inference(client):
- response = client.post("/v1/cron-tasks/run")
- assert response.status_code == status.HTTP_200_OK
-
-
-@pytest.mark.order(get_order_number("cron_tasks_run_ok"))
-def test_cron_tasks_ok(client):
- response = client.post("/v1/cron-tasks/run")
- assert response.status_code == status.HTTP_200_OK
-
-
-@pytest.mark.order(get_order_number("cron_tasks_run_after_x_time"))
-def test_cron_tasks_run_after_x_time(client):
- response = client.post("/v1/cron-tasks/run")
- assert response.status_code == status.HTTP_200_OK
diff --git a/whitebox/tests/v1/test_dataset_rows.py b/whitebox/tests/v1/test_dataset_rows.py
deleted file mode 100644
index a12cba9..0000000
--- a/whitebox/tests/v1/test_dataset_rows.py
+++ /dev/null
@@ -1,125 +0,0 @@
-from whitebox.tests.v1.mock_data import (
- dataset_rows_single_row_column_payload,
- dataset_rows_no_prediction_column_payload,
- dataset_rows_one_prediction_value_payload,
- dataset_rows_create_multi_class_payload,
- dataset_rows_create_binary_payload,
- dataset_rows_create_wrong_model_payload,
- dataset_rows_create_reg_payload,
-)
-import pytest
-from whitebox import schemas
-from whitebox.tests.v1.conftest import get_order_number, state
-from fastapi import status
-
-
-@pytest.mark.order(get_order_number("dataset_rows_wrong_training_dataset"))
-def test_dataset_rows_wrong_training_data(client, api_key):
- response_single_row = client.post(
- "/v1/dataset-rows",
- json=list(
- map(
- lambda x: {**x, "model_id": state.model_multi["id"]},
- dataset_rows_single_row_column_payload,
- )
- ),
- headers={"api-key": api_key},
- )
-
- response_no_prediction = client.post(
- "/v1/dataset-rows",
- json=list(
- map(
- lambda x: {**x, "model_id": state.model_multi["id"]},
- dataset_rows_no_prediction_column_payload,
- )
- ),
- headers={"api-key": api_key},
- )
-
- response_one_prediction_value = client.post(
- "/v1/dataset-rows",
- json=list(
- map(
- lambda x: {**x, "model_id": state.model_multi["id"]},
- dataset_rows_one_prediction_value_payload,
- )
- ),
- headers={"api-key": api_key},
- )
-
- assert response_single_row.status_code == status.HTTP_400_BAD_REQUEST
- assert response_no_prediction.status_code == status.HTTP_400_BAD_REQUEST
- assert response_one_prediction_value.status_code == status.HTTP_400_BAD_REQUEST
-
-
-@pytest.mark.order(get_order_number("dataset_rows_create"))
-def test_dataset_row_create_many(client, api_key):
- response_model_multi = client.post(
- "/v1/dataset-rows",
- json=list(
- map(
- lambda x: {**x, "model_id": state.model_multi["id"]},
- dataset_rows_create_multi_class_payload,
- )
- ),
- headers={"api-key": api_key},
- )
-
- response_model_binary = client.post(
- "/v1/dataset-rows",
- json=list(
- map(
- lambda x: {**x, "model_id": state.model_binary["id"]},
- dataset_rows_create_binary_payload,
- )
- ),
- headers={"api-key": api_key},
- )
-
- response_model_reg = client.post(
- "/v1/dataset-rows",
- json=list(
- map(
- lambda x: {**x, "model_id": state.model_regression["id"]},
- dataset_rows_create_reg_payload,
- )
- ),
- headers={"api-key": api_key},
- )
-
- response_wrong_model = client.post(
- "/v1/dataset-rows",
- json=(dataset_rows_create_wrong_model_payload),
- headers={"api-key": api_key},
- )
-
- assert response_model_multi.status_code == status.HTTP_201_CREATED
- assert response_model_binary.status_code == status.HTTP_201_CREATED
- assert response_model_reg.status_code == status.HTTP_201_CREATED
- assert response_wrong_model.status_code == status.HTTP_404_NOT_FOUND
- validated = [schemas.DatasetRow(**m) for m in response_model_multi.json()]
- validated = [schemas.DatasetRow(**m) for m in response_model_binary.json()]
- validated = [schemas.DatasetRow(**m) for m in response_model_reg.json()]
-
-
-@pytest.mark.order(get_order_number("dataset_rows_get_model's_all"))
-def test_dataset_row_get_models_all(client, api_key):
- response_model_multi = client.get(
- f"/v1/dataset-rows?model_id={state.model_multi['id']}",
- headers={"api-key": api_key},
- )
- response_model_binary = client.get(
- f"/v1/dataset-rows?model_id={state.model_binary['id']}",
- headers={"api-key": api_key},
- )
- response_model_not_found = client.get(
- f"/v1/dataset-rows?model_id=wrong_model_id",
- headers={"api-key": api_key},
- )
-
- assert response_model_multi.status_code == status.HTTP_200_OK
- assert response_model_binary.status_code == status.HTTP_200_OK
- assert response_model_not_found.status_code == status.HTTP_404_NOT_FOUND
- validated = [schemas.DatasetRow(**m) for m in response_model_multi.json()]
- validated = [schemas.DatasetRow(**m) for m in response_model_binary.json()]
diff --git a/whitebox/tests/v1/test_drifting_metrics.py b/whitebox/tests/v1/test_drifting_metrics.py
deleted file mode 100644
index 3939a94..0000000
--- a/whitebox/tests/v1/test_drifting_metrics.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import pytest
-from whitebox import schemas
-from whitebox.tests.v1.conftest import get_order_number, state
-from fastapi import status
-
-
-@pytest.mark.order(get_order_number("drifting_metrics_get_model_all"))
-def test_drifting_metric_get_model_all(client, api_key):
- response_multi = client.get(
- f"/v1/drifting-metrics?model_id={state.model_multi['id']}",
- headers={"api-key": api_key},
- )
-
- response_binary = client.get(
- f"/v1/drifting-metrics?model_id={state.model_binary['id']}",
- headers={"api-key": api_key},
- )
- response_wrong_model = client.get(
- f"/v1/drifting-metrics?model_id=wrong_model_id",
- headers={"api-key": api_key},
- )
-
- response_binary_json = response_binary.json()
- response_multi_json = response_multi.json()
-
- assert len(response_multi_json) == 1
- assert len(response_binary_json) == 1
-
- assert response_multi_json[0]["timestamp"] == "2023-03-06T12:15:00"
- assert response_binary_json[0]["timestamp"] == "2023-03-07T00:00:00"
-
- assert response_multi.status_code == status.HTTP_200_OK
- assert response_binary.status_code == status.HTTP_200_OK
- assert response_wrong_model.status_code == status.HTTP_404_NOT_FOUND
-
- validated = [schemas.DriftingMetric(**m) for m in response_multi_json]
- validated = [schemas.DriftingMetric(**m) for m in response_binary_json]
-
-
-@pytest.mark.order(get_order_number("drifting_metrics_get_binary_model_after_x_time"))
-def test_drifting_metrics_get_binary_model_after_x_time(client, api_key):
- response_binary = client.get(
- f"/v1/drifting-metrics?model_id={state.model_binary['id']}",
- headers={"api-key": api_key},
- )
-
- response_binary_json = response_binary.json()
-
- assert len(response_binary_json) == 1
-
- assert response_binary_json[0]["timestamp"] == "2023-03-07T00:00:00"
-
- assert response_binary.status_code == status.HTTP_200_OK
-
- validated = [schemas.DriftingMetric(**m) for m in response_binary_json]
diff --git a/whitebox/tests/v1/test_errors.py b/whitebox/tests/v1/test_errors.py
deleted file mode 100644
index f486d82..0000000
--- a/whitebox/tests/v1/test_errors.py
+++ /dev/null
@@ -1,20 +0,0 @@
-import pytest
-from whitebox.tests.v1.conftest import get_order_number
-from fastapi import status
-
-
-@pytest.mark.order(get_order_number("models_no_api_key"))
-def test_model_no_api_key(client):
- response = client.get("/v1/models")
-
- assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY
-
-
-@pytest.mark.order(get_order_number("models_wrong_api_key"))
-def test_model_wrong_api_key(client):
- response = client.get(
- "/v1/models",
- headers={"api-key": "1234567890"},
- )
-
- assert response.status_code == status.HTTP_401_UNAUTHORIZED
diff --git a/whitebox/tests/v1/test_health.py b/whitebox/tests/v1/test_health.py
deleted file mode 100644
index 7a85a23..0000000
--- a/whitebox/tests/v1/test_health.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import pytest
-from whitebox import schemas
-from whitebox.tests.v1.conftest import get_order_number
-from fastapi import status
-
-
-@pytest.mark.order(get_order_number("health"))
-def test_health(client):
- response = client.get("/v1/health")
- assert response.status_code == status.HTTP_200_OK
- validated = schemas.HealthCheck(**response.json())
diff --git a/whitebox/tests/v1/test_inference_rows.py b/whitebox/tests/v1/test_inference_rows.py
deleted file mode 100644
index 320e930..0000000
--- a/whitebox/tests/v1/test_inference_rows.py
+++ /dev/null
@@ -1,191 +0,0 @@
-from whitebox.tests.v1.mock_data import (
- inference_row_create_single_row_payload,
- inference_row_create_many_binary_payload,
- inference_row_create_many_multi_payload,
- inference_row_create_many_multi_no_actual_payload,
- inference_row_create_many_multi_mixed_actuals_payload,
- inference_row_create_many_reg_payload,
-)
-import pytest
-from whitebox import schemas
-from whitebox.tests.v1.conftest import get_order_number, state
-from fastapi import status
-
-
-@pytest.mark.order(get_order_number("inference_rows_create"))
-def test_inference_row_create(client, api_key):
- response = client.post(
- "/v1/inference-rows",
- json={
- **inference_row_create_single_row_payload,
- "model_id": state.model_multi["id"],
- },
- headers={"api-key": api_key},
- )
- state.inference_row_multi = response.json()
- assert response.status_code == status.HTTP_201_CREATED
- validated = schemas.InferenceRow(**response.json())
-
-
-@pytest.mark.order(get_order_number("inference_rows_create_many"))
-def test_inference_row_create_many(client, api_key):
- response_binary = client.post(
- "/v1/inference-rows/batch",
- json=list(
- map(
- lambda x: {**x, "model_id": state.model_binary["id"]},
- inference_row_create_many_binary_payload,
- )
- ),
- headers={"api-key": api_key},
- )
-
- state.inference_row_binary = response_binary.json()[0]
-
- response_multi = client.post(
- "/v1/inference-rows/batch",
- json=list(
- map(
- lambda x: {**x, "model_id": state.model_multi["id"]},
- inference_row_create_many_multi_payload,
- )
- ),
- headers={"api-key": api_key},
- )
-
- response_multi_2 = client.post(
- "/v1/inference-rows/batch",
- json=list(
- map(
- lambda x: {**x, "model_id": state.model_multi_2["id"]},
- inference_row_create_many_multi_no_actual_payload,
- )
- ),
- headers={"api-key": api_key},
- )
-
- response_multi_3 = client.post(
- "/v1/inference-rows/batch",
- json=list(
- map(
- lambda x: {**x, "model_id": state.model_multi_3["id"]},
- inference_row_create_many_multi_mixed_actuals_payload,
- )
- ),
- headers={"api-key": api_key},
- )
-
- response_reg = client.post(
- "/v1/inference-rows/batch",
- json=list(
- map(
- lambda x: {**x, "model_id": state.model_regression["id"]},
- inference_row_create_many_reg_payload,
- )
- ),
- headers={"api-key": api_key},
- )
-
- assert response_binary.status_code == status.HTTP_201_CREATED
- assert response_multi.status_code == status.HTTP_201_CREATED
- assert response_reg.status_code == status.HTTP_201_CREATED
- validated = [schemas.InferenceRow(**m) for m in response_binary.json()]
- validated = [schemas.InferenceRow(**m) for m in response_multi.json()]
- validated = [schemas.InferenceRow(**m) for m in response_multi_2.json()]
- validated = [schemas.InferenceRow(**m) for m in response_multi_3.json()]
- validated = [schemas.InferenceRow(**m) for m in response_reg.json()]
-
-
-@pytest.mark.order(get_order_number("inference_rows_get_model's_all"))
-def test_inference_row_get_models_all(client, api_key):
- response_multi = client.get(
- f"/v1/inference-rows?model_id={state.model_multi['id']}",
- headers={"api-key": api_key},
- )
- response_binary = client.get(
- f"/v1/inference-rows?model_id={state.model_binary['id']}",
- headers={"api-key": api_key},
- )
- response_wrong_model = client.get(
- f"/v1/inference-rows?model_id=wrong_model_id",
- headers={"api-key": api_key},
- )
- assert response_multi.status_code == status.HTTP_200_OK
- assert response_binary.status_code == status.HTTP_200_OK
- assert response_wrong_model.status_code == status.HTTP_404_NOT_FOUND
- validated = [schemas.InferenceRow(**m) for m in response_multi.json()]
- validated = [schemas.InferenceRow(**m) for m in response_binary.json()]
-
-
-@pytest.mark.order(get_order_number("inference_rows_get"))
-def test_inference_row_get(client, api_key):
- response = client.get(
- f"/v1/inference-rows/{state.inference_row_multi['id']}",
- headers={"api-key": api_key},
- )
- response_wrong_inference = client.get(
- f"/v1/inference-rows/wrong_inference_id",
- headers={"api-key": api_key},
- )
- assert response.status_code == status.HTTP_200_OK
- assert response_wrong_inference.status_code == status.HTTP_404_NOT_FOUND
- validated = schemas.InferenceRow(**response.json())
-
-
-@pytest.mark.order(get_order_number("inference_rows_xai"))
-def test_inference_row_xai(client, api_key):
- response = client.get(
- f"/v1/inference-rows/{state.inference_row_binary['id']}/xai",
- headers={"api-key": api_key},
- )
-
- response_wrong_inference = client.get(
- f"/v1/inference-rows/wrong_inference_id/xai",
- headers={"api-key": api_key},
- )
-
- assert response.status_code == status.HTTP_200_OK
- assert response_wrong_inference.status_code == status.HTTP_404_NOT_FOUND
-
-
-@pytest.mark.order(get_order_number("inference_rows_create_many_after_x_time"))
-def test_inference_row_create_many_after_x_time(client, api_key):
- response_binary = client.post(
- "/v1/inference-rows/batch",
- json=list(
- map(
- lambda x: {**x, "model_id": state.model_binary["id"]},
- inference_row_create_many_binary_payload,
- )
- ),
- headers={"api-key": api_key},
- )
-
- response_multi = client.post(
- "/v1/inference-rows/batch",
- json=list(
- map(
- lambda x: {**x, "model_id": state.model_multi["id"]},
- inference_row_create_many_multi_payload,
- )
- ),
- headers={"api-key": api_key},
- )
-
- response_reg = client.post(
- "/v1/inference-rows/batch",
- json=list(
- map(
- lambda x: {**x, "model_id": state.model_regression["id"]},
- inference_row_create_many_reg_payload,
- )
- ),
- headers={"api-key": api_key},
- )
-
- assert response_binary.status_code == status.HTTP_201_CREATED
- assert response_multi.status_code == status.HTTP_201_CREATED
- assert response_reg.status_code == status.HTTP_201_CREATED
- validated = [schemas.InferenceRow(**m) for m in response_binary.json()]
- validated = [schemas.InferenceRow(**m) for m in response_multi.json()]
- validated = [schemas.InferenceRow(**m) for m in response_reg.json()]
diff --git a/whitebox/tests/v1/test_model_integrity_metrics.py b/whitebox/tests/v1/test_model_integrity_metrics.py
deleted file mode 100644
index f7378bb..0000000
--- a/whitebox/tests/v1/test_model_integrity_metrics.py
+++ /dev/null
@@ -1,30 +0,0 @@
-import pytest
-from whitebox import schemas
-from whitebox.tests.v1.conftest import get_order_number, state
-from fastapi import status
-
-
-@pytest.mark.order(get_order_number("model_integrity_metrics_get_model_all"))
-def test_model_integrity_metric_get_model_all(client, api_key):
- response_multi = client.get(
- f"/v1/model-integrity-metrics?model_id={state.model_multi['id']}",
- headers={"api-key": api_key},
- )
- response_binary = client.get(
- f"/v1/model-integrity-metrics?model_id={state.model_binary['id']}",
- headers={"api-key": api_key},
- )
- response_wrong_model = client.get(
- f"/v1/model-integrity-metrics?model_id=wrong_model_id",
- headers={"api-key": api_key},
- )
-
- assert len(response_multi.json()) == 1
- assert len(response_binary.json()) == 1
-
- assert response_multi.status_code == status.HTTP_200_OK
- assert response_binary.status_code == status.HTTP_200_OK
- assert response_wrong_model.status_code == status.HTTP_404_NOT_FOUND
-
- validated = [schemas.ModelIntegrityMetric(**m) for m in response_multi.json()]
- validated = [schemas.ModelIntegrityMetric(**m) for m in response_binary.json()]
diff --git a/whitebox/tests/v1/test_model_monitors.py b/whitebox/tests/v1/test_model_monitors.py
deleted file mode 100644
index 4183e33..0000000
--- a/whitebox/tests/v1/test_model_monitors.py
+++ /dev/null
@@ -1,206 +0,0 @@
-import pytest
-from whitebox.tests.v1.mock_data import (
- model_monitor_accuracy_create_payload,
- model_monitor_f1_create_payload,
- model_monitor_data_drift_create_payload,
- model_monitor_concept_drift_create_payload,
- model_monitor_precision_create_payload,
- model_monitor_r_square_create_payload,
- model_monitor_no_threshold_create_payload,
- model_monitor_no_feature_create_payload,
- model_monitor_feature_same_as_target_create_payload,
- model_monitor_feature_not_in_columns_create_payload,
- model_monitor_update_payload,
-)
-from whitebox import schemas
-from whitebox.tests.v1.conftest import get_order_number, state
-from fastapi import status
-
-
-@pytest.mark.order(get_order_number("model_monitor_create"))
-def test_model_monitor_create(client, api_key):
- accuracy_monitor = client.post(
- "/v1/model-monitors",
- json={
- **model_monitor_accuracy_create_payload,
- "model_id": state.model_multi["id"],
- },
- headers={"api-key": api_key},
- )
-
- f1_monitor = client.post(
- "/v1/model-monitors",
- json={
- **model_monitor_f1_create_payload,
- "model_id": state.model_multi["id"],
- },
- headers={"api-key": api_key},
- )
-
- data_drift_monitor = client.post(
- "/v1/model-monitors",
- json={
- **model_monitor_data_drift_create_payload,
- "model_id": state.model_binary["id"],
- },
- headers={"api-key": api_key},
- )
-
- concept_drift_monitor = client.post(
- "/v1/model-monitors",
- json={
- **model_monitor_concept_drift_create_payload,
- "model_id": state.model_binary["id"],
- },
- headers={"api-key": api_key},
- )
-
- precision_monitor = client.post(
- "/v1/model-monitors",
- json={
- **model_monitor_precision_create_payload,
- "model_id": state.model_binary["id"],
- },
- headers={"api-key": api_key},
- )
-
- r_square_monitor = client.post(
- "/v1/model-monitors",
- json={
- **model_monitor_r_square_create_payload,
- "model_id": state.model_regression["id"],
- },
- headers={"api-key": api_key},
- )
-
- wrong_model_monitor = client.post(
- "/v1/model-monitors",
- json={
- **model_monitor_r_square_create_payload,
- "model_id": "wrong_model_id",
- },
- headers={"api-key": api_key},
- )
-
- no_training_data = client.post(
- "/v1/model-monitors",
- json={
- **model_monitor_data_drift_create_payload,
- "model_id": state.model_multi_3["id"],
- },
- headers={"api-key": api_key},
- )
-
- no_threshold = client.post(
- "/v1/model-monitors",
- json={
- **model_monitor_no_threshold_create_payload,
- "model_id": state.model_multi["id"],
- },
- headers={"api-key": api_key},
- )
-
- no_feature = client.post(
- "/v1/model-monitors",
- json={
- **model_monitor_no_feature_create_payload,
- "model_id": state.model_multi["id"],
- },
- headers={"api-key": api_key},
- )
-
- feature_same_as_target = client.post(
- "/v1/model-monitors",
- json={
- **model_monitor_feature_same_as_target_create_payload,
- "model_id": state.model_multi["id"],
- },
- headers={"api-key": api_key},
- )
-
- feature_not_in_columns = client.post(
- "/v1/model-monitors",
- json={
- **model_monitor_feature_not_in_columns_create_payload,
- "model_id": state.model_multi["id"],
- },
- headers={"api-key": api_key},
- )
-
- state.concept_drift_monitor = (
- concept_drift_monitor_json
- ) = concept_drift_monitor.json()
-
- assert concept_drift_monitor.status_code == status.HTTP_201_CREATED
- assert concept_drift_monitor_json["feature"] == "target"
- assert wrong_model_monitor.status_code == status.HTTP_404_NOT_FOUND
- assert no_threshold.status_code == status.HTTP_400_BAD_REQUEST
- assert no_feature.status_code == status.HTTP_400_BAD_REQUEST
- assert feature_same_as_target.status_code == status.HTTP_400_BAD_REQUEST
- assert feature_not_in_columns.status_code == status.HTTP_400_BAD_REQUEST
- assert no_training_data.status_code == status.HTTP_404_NOT_FOUND
- validated = schemas.ModelMonitor(**concept_drift_monitor_json)
-
-
-@pytest.mark.order(get_order_number("model_monitors_get_model_all"))
-def test_model_monitors_get_model_all(client, api_key):
- response_multi = client.get(
- f"/v1/model-monitors?model_id={state.model_multi['id']}",
- headers={"api-key": api_key},
- )
- response_all = client.get(
- f"/v1/model-monitors",
- headers={"api-key": api_key},
- )
- response_wrong_model = client.get(
- f"/v1/model-monitors?model_id=wrong_model_id",
- headers={"api-key": api_key},
- )
-
- assert len(response_multi.json()) == 2
- assert len(response_all.json()) == 6
-
- assert response_multi.status_code == status.HTTP_200_OK
- assert response_all.status_code == status.HTTP_200_OK
- assert response_wrong_model.status_code == status.HTTP_404_NOT_FOUND
-
- validated = [schemas.ModelMonitor(**m) for m in response_multi.json()]
- validated = [schemas.ModelMonitor(**m) for m in response_all.json()]
-
-
-@pytest.mark.order(get_order_number("model_monitor_update"))
-def test_model_monitor_update(client, api_key):
- response = client.put(
- f"/v1/model-monitors/{state.concept_drift_monitor['id']}",
- json=model_monitor_update_payload,
- headers={"api-key": api_key},
- )
- response_wrong_model = client.put(
- f"/v1/model-monitors/wrong_model_id",
- json=model_monitor_update_payload,
- headers={"api-key": api_key},
- )
-
- response_json = response.json()
-
- assert response.status_code == status.HTTP_200_OK
- assert response_json["lower_threshold"] == None
- assert response_wrong_model.status_code == status.HTTP_404_NOT_FOUND
-
- validated = schemas.ModelMonitor(**response_json)
-
-
-@pytest.mark.order(get_order_number("model_monitor_delete"))
-def test_model_monitor_delete(client, api_key):
- response = client.delete(
- f"/v1/model-monitors/{state.concept_drift_monitor['id']}",
- headers={"api-key": api_key},
- )
-
- wrong_monitor_response = client.delete(
- f"/v1/model-monitors/wrong_model_monitor_id",
- headers={"api-key": api_key},
- )
-
- assert response.status_code == status.HTTP_200_OK
- assert wrong_monitor_response.status_code == status.HTTP_404_NOT_FOUND
diff --git a/whitebox/tests/v1/test_models.py b/whitebox/tests/v1/test_models.py
deleted file mode 100644
index e7aaeff..0000000
--- a/whitebox/tests/v1/test_models.py
+++ /dev/null
@@ -1,133 +0,0 @@
-from whitebox.tests.v1.mock_data import (
- model_binary_create_payload,
- model_multi_create_payload,
- model_multi_2_create_payload,
- model_multi_3_create_payload,
- model_regression_create_payload,
- model_update_payload,
-)
-import pytest
-from whitebox import schemas
-from whitebox.tests.v1.conftest import get_order_number, state
-from fastapi import status
-
-
-@pytest.mark.order(get_order_number("models_create"))
-def test_model_create(client, api_key):
- response_binary = client.post(
- "/v1/models",
- json={**model_binary_create_payload},
- headers={"api-key": api_key},
- )
-
- response_multi = client.post(
- "/v1/models",
- json={**model_multi_create_payload},
- headers={"api-key": api_key},
- )
-
- response_multi_2 = client.post(
- "/v1/models",
- json={**model_multi_2_create_payload},
- headers={"api-key": api_key},
- )
-
- response_multi_3 = client.post(
- "/v1/models",
- json={**model_multi_3_create_payload},
- headers={"api-key": api_key},
- )
-
- response_regression = client.post(
- "/v1/models",
- json={**model_regression_create_payload},
- headers={"api-key": api_key},
- )
-
- state.model_binary = response_binary.json()
- state.model_multi = response_multi.json()
- state.model_multi_2 = response_multi_2.json()
- state.model_multi_3 = response_multi_3.json()
- state.model_regression = response_regression.json()
-
- assert response_binary.status_code == status.HTTP_201_CREATED
- assert response_multi.status_code == status.HTTP_201_CREATED
- assert response_multi_2.status_code == status.HTTP_201_CREATED
- assert response_multi_3.status_code == status.HTTP_201_CREATED
- assert response_regression.status_code == status.HTTP_201_CREATED
-
- validated = schemas.Model(**response_binary.json())
- validated = schemas.Model(**response_multi.json())
- validated = schemas.Model(**response_multi_2.json())
- validated = schemas.Model(**response_multi_3.json())
- validated = schemas.Model(**response_regression.json())
-
-
-@pytest.mark.order(get_order_number("models_get_all"))
-def test_model_get_all(client, api_key):
- response = client.get(f"/v1/models", headers={"api-key": api_key})
- assert response.status_code == status.HTTP_200_OK
- validated = [schemas.Model(**m) for m in response.json()]
-
-
-@pytest.mark.order(get_order_number("models_get"))
-def test_model_get(client, api_key):
- response = client.get(
- f"/v1/models/{state.model_multi['id']}", headers={"api-key": api_key}
- )
- response_wrong_model = client.get(
- f"/v1/models/wrong_model_id", headers={"api-key": api_key}
- )
-
- assert response.status_code == status.HTTP_200_OK
- assert response_wrong_model.status_code == status.HTTP_404_NOT_FOUND
-
- validated = schemas.Model(**response.json())
-
-
-@pytest.mark.order(get_order_number("models_update"))
-def test_model_update(client, api_key):
- response = client.put(
- f"/v1/models/{state.model_multi['id']}",
- json=model_update_payload,
- headers={"api-key": api_key},
- )
- response_wrong_model = client.put(
- f"/v1/models/wrong_model_id",
- json=model_update_payload,
- headers={"api-key": api_key},
- )
-
- assert response.status_code == status.HTTP_200_OK
- assert response_wrong_model.status_code == status.HTTP_404_NOT_FOUND
-
- validated = schemas.Model(**response.json())
-
-
-@pytest.mark.order(get_order_number("models_delete"))
-def test_model_delete(client, api_key):
- response_binary = client.delete(
- f"/v1/models/{state.model_binary['id']}", headers={"api-key": api_key}
- )
- response_multi = client.delete(
- f"/v1/models/{state.model_multi['id']}", headers={"api-key": api_key}
- )
- response_multi_2 = client.delete(
- f"/v1/models/{state.model_multi_2['id']}", headers={"api-key": api_key}
- )
- response_multi_3 = client.delete(
- f"/v1/models/{state.model_multi_3['id']}", headers={"api-key": api_key}
- )
- response_regression = client.delete(
- f"/v1/models/{state.model_regression['id']}", headers={"api-key": api_key}
- )
- response_no_model = client.delete(
- f"/v1/models/{state.model_binary['id']}", headers={"api-key": api_key}
- )
-
- assert response_binary.status_code == status.HTTP_200_OK
- assert response_multi.status_code == status.HTTP_200_OK
- assert response_multi_2.status_code == status.HTTP_200_OK
- assert response_multi_3.status_code == status.HTTP_200_OK
- assert response_regression.status_code == status.HTTP_200_OK
- assert response_no_model.status_code == status.HTTP_404_NOT_FOUND
diff --git a/whitebox/tests/v1/test_performance_metrics.py b/whitebox/tests/v1/test_performance_metrics.py
deleted file mode 100644
index 690527d..0000000
--- a/whitebox/tests/v1/test_performance_metrics.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import pytest
-from whitebox import schemas
-from whitebox.tests.v1.conftest import get_order_number, state
-from fastapi import status
-
-
-@pytest.mark.order(get_order_number("performance_metrics_get_model_all"))
-def test_performance_metric_get_model_all(client, api_key):
- response_multi = client.get(
- f"/v1/performance-metrics?model_id={state.model_multi['id']}",
- headers={"api-key": api_key},
- )
- response_binary = client.get(
- f"/v1/performance-metrics?model_id={state.model_binary['id']}",
- headers={"api-key": api_key},
- )
- response_reg = client.get(
- f"/v1/performance-metrics?model_id={state.model_regression['id']}",
- headers={"api-key": api_key},
- )
- response_wrong_model = client.get(
- f"/v1/performance-metrics?model_id=wrong_model_id",
- headers={"api-key": api_key},
- )
-
- assert response_multi.status_code == status.HTTP_200_OK
- assert response_binary.status_code == status.HTTP_200_OK
- assert response_reg.status_code == status.HTTP_200_OK
- assert response_wrong_model.status_code == status.HTTP_404_NOT_FOUND
-
- validated = [schemas.MultiClassificationMetrics(**m) for m in response_multi.json()]
- validated = [
- schemas.BinaryClassificationMetrics(**m) for m in response_binary.json()
- ]
- validated = [schemas.RegressionMetrics(**m) for m in response_reg.json()]
diff --git a/whitebox/tests/v1/test_sdk.py b/whitebox/tests/v1/test_sdk.py
deleted file mode 100644
index 7ef4994..0000000
--- a/whitebox/tests/v1/test_sdk.py
+++ /dev/null
@@ -1,397 +0,0 @@
-import pandas as pd
-import pytest
-from whitebox.schemas.modelMonitor import AlertSeverity, MonitorMetrics, MonitorStatus
-from whitebox.sdk import Whitebox
-from whitebox.tests.v1.conftest import get_order_number, state, state_sdk
-from whitebox.tests.v1.mock_data import (
- model_multi_create_payload,
- timestamps,
- mixed_actuals,
- inference_row_xai_payload,
- model_monitor_concept_drift_create_payload,
- model_monitor_update_payload,
- alert_payload,
- drifting_metrics_report_payload,
- descriptive_statistics_report_payload,
- performance_metrics_report_payload,
-)
-import requests_mock
-from fastapi import status
-
-
-@pytest.mark.order(get_order_number("sdk_init"))
-def test_sdk_init(client, api_key):
- wb = Whitebox(host=client.base_url, api_key=api_key)
- assert wb.host == client.base_url
- assert wb.api_key == api_key
- state_sdk.wb = wb
-
-
-@pytest.mark.order(get_order_number("sdk_create_model"))
-def test_sdk_create_model(client):
- with requests_mock.Mocker() as m:
- m.post(
- url=f"{state_sdk.wb.host}/v1/models",
- json=model_multi_create_payload,
- headers={"api-key": state_sdk.wb.api_key},
- )
-
- model = state_sdk.wb.create_model(
- name=model_multi_create_payload["name"],
- description=model_multi_create_payload["description"],
- labels=model_multi_create_payload["labels"],
- type=model_multi_create_payload["type"],
- target_column=model_multi_create_payload["target_column"],
- granularity=model_multi_create_payload["granularity"],
- )
-
- assert model == model_multi_create_payload
-
-
-@pytest.mark.order(get_order_number("sdk_get_model"))
-def test_sdk_get_model(client):
- mock_model_id = "mock_model_id"
- with requests_mock.Mocker() as m:
- m.get(
- url=f"{state_sdk.wb.host}/v1/models/{mock_model_id}",
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_404_NOT_FOUND,
- )
-
- not_found_result = state_sdk.wb.get_model(model_id=mock_model_id)
-
- assert not_found_result == None
-
- m.get(
- url=f"{state_sdk.wb.host}/v1/models/{mock_model_id}",
- headers={"api-key": state_sdk.wb.api_key},
- json=state.model_multi,
- )
-
- model = state_sdk.wb.get_model(model_id=mock_model_id)
-
- assert model == state.model_multi
-
-
-@pytest.mark.order(get_order_number("sdk_delete_model"))
-def test_sdk_delete_model(client):
- mock_model_id = "mock_model_id"
-
- with requests_mock.Mocker() as m:
- m.delete(
- url=f"{state_sdk.wb.host}/v1/models/{mock_model_id}",
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_200_OK,
- )
-
- happy_result = state_sdk.wb.delete_model(model_id=mock_model_id)
- assert happy_result == True
-
- m.delete(
- url=f"{state_sdk.wb.host}/v1/models/{mock_model_id}",
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_400_BAD_REQUEST,
- )
-
- sad_result = state_sdk.wb.delete_model(model_id=mock_model_id)
- assert sad_result == False
-
-
-@pytest.mark.order(get_order_number("sdk_log_training_dataset"))
-def test_sdk_log_training_dataset(client):
- mock_model_id = "mock_model_id"
- df = pd.read_csv("whitebox/analytics/data/testing/classification_test_data.csv")
-
- with requests_mock.Mocker() as m:
- m.post(
- url=f"{state_sdk.wb.host}/v1/dataset-rows",
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_201_CREATED,
- )
-
- happy_result = state_sdk.wb.log_training_dataset(
- model_id=mock_model_id, processed=df, non_processed=df
- )
- assert happy_result == True
-
- m.post(
- url=f"{state_sdk.wb.host}/v1/dataset-rows",
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_400_BAD_REQUEST,
- )
-
- sad_result = state_sdk.wb.log_training_dataset(
- model_id=mock_model_id, processed=df, non_processed=df
- )
- assert sad_result == False
-
- # drop a row in df to test the dataframe length error handling
- df2 = df.drop(df.index[0])
- with pytest.raises(Exception) as e_info:
- state_sdk.wb.log_inferences(
- model_id=mock_model_id, processed=df, non_processed=df2
- )
-
-
-@pytest.mark.order(get_order_number("sdk_log_inferences"))
-def test_sdk_log_inferences(client):
- mock_model_id = "mock_model_id"
- df = pd.read_csv("whitebox/analytics/data/testing/classification_test_data.csv")
-
- with requests_mock.Mocker() as m:
- m.post(
- url=f"{state_sdk.wb.host}/v1/inference-rows/batch",
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_201_CREATED,
- )
-
- happy_result = state_sdk.wb.log_inferences(
- model_id=mock_model_id,
- processed=df,
- non_processed=df,
- timestamps=timestamps,
- actuals=mixed_actuals,
- )
- assert happy_result == True
-
- m.post(
- url=f"{state_sdk.wb.host}/v1/inference-rows/batch",
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_400_BAD_REQUEST,
- )
-
- sad_result = state_sdk.wb.log_inferences(
- model_id=mock_model_id,
- processed=df,
- non_processed=df,
- timestamps=timestamps,
- )
- assert sad_result == False
-
- # drop a row in df to test the dataframe length error handling
- df2 = df.drop(df.index[0])
- with pytest.raises(Exception) as e_info:
- state_sdk.wb.log_inferences(
- model_id=mock_model_id,
- processed=df,
- non_processed=df2,
- timestamps=timestamps,
- actuals=mixed_actuals,
- )
-
-
-@pytest.mark.order(get_order_number("sdk_create_model_monitor"))
-def test_sdk_create_model_monitor(client):
- with requests_mock.Mocker() as m:
- m.post(
- url=f"{state_sdk.wb.host}/v1/model-monitors",
- json=model_monitor_concept_drift_create_payload,
- headers={"api-key": state_sdk.wb.api_key},
- )
-
- model_monitor = state_sdk.wb.create_model_monitor(
- model_id="mock_model_id",
- name="test",
- status=MonitorStatus.active,
- metric=MonitorMetrics.accuracy,
- feature="feature1",
- lower_threshold=0.7,
- severity=AlertSeverity.high,
- email="jackie.chan@chinamail.io",
- )
-
- assert model_monitor is not None
-
-
-@pytest.mark.order(get_order_number("sdk_update_model_monitor"))
-def test_sdk_update_model_monitor(client):
- mock_model_monitor_id = "mock_model_monitor_id"
-
- with requests_mock.Mocker() as m:
- m.put(
- url=f"{state_sdk.wb.host}/v1/model-monitors/{mock_model_monitor_id}",
- json=model_monitor_update_payload,
- headers={"api-key": state_sdk.wb.api_key},
- )
-
- happy_result = state_sdk.wb.update_model_monitor(
- model_monitor_id=mock_model_monitor_id,
- name="concept drift monitor updated",
- lower_threshold=0.54,
- )
-
- assert happy_result == True
-
- m.put(
- url=f"{state_sdk.wb.host}/v1/model-monitors/{mock_model_monitor_id}",
- json=model_monitor_update_payload,
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_400_BAD_REQUEST,
- )
-
- sad_result = state_sdk.wb.update_model_monitor(
- model_monitor_id=mock_model_monitor_id,
- name="concept drift monitor updated",
- lower_threshold=0.54,
- )
-
- assert sad_result == False
-
-
-@pytest.mark.order(get_order_number("sdk_delete_model_monitor"))
-def test_sdk_delete_model_monitor(client):
- mock_model_monitor_id = "mock_model_monitor_id"
-
- with requests_mock.Mocker() as m:
- m.delete(
- url=f"{state_sdk.wb.host}/v1/model-monitors/{mock_model_monitor_id}",
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_200_OK,
- )
-
- happy_result = state_sdk.wb.delete_model_monitor(
- model_monitor_id=mock_model_monitor_id
- )
- assert happy_result == True
-
- m.delete(
- url=f"{state_sdk.wb.host}/v1/model-monitors/{mock_model_monitor_id}",
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_400_BAD_REQUEST,
- )
-
- sad_result = state_sdk.wb.delete_model_monitor(
- model_monitor_id=mock_model_monitor_id
- )
- assert sad_result == False
-
-
-@pytest.mark.order(get_order_number("sdk_get_alerts"))
-def test_sdk_get_alerts(client):
- mock_model_id = "mock_model_id"
- with requests_mock.Mocker() as m:
- m.get(
- url=f"{state_sdk.wb.host}/v1/alerts?model_id={mock_model_id}",
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_404_NOT_FOUND,
- )
-
- not_found_result = state_sdk.wb.get_alerts(model_id=mock_model_id)
-
- assert not_found_result == None
-
- m.get(
- url=f"{state_sdk.wb.host}/v1/alerts?model_id={mock_model_id}",
- headers={"api-key": state_sdk.wb.api_key},
- json=alert_payload,
- )
-
- alert = state_sdk.wb.get_alerts(model_id=mock_model_id)
-
- assert alert == alert_payload
-
-
-@pytest.mark.order(get_order_number("sdk_get_drifting_metrics"))
-def test_sdk_get_drifting_metrics(client):
- mock_model_id = "mock_model_id"
- with requests_mock.Mocker() as m:
- m.get(
- url=f"{state_sdk.wb.host}/v1/drifting-metrics?model_id={mock_model_id}",
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_404_NOT_FOUND,
- )
-
- not_found_result = state_sdk.wb.get_drifting_metrics(model_id=mock_model_id)
-
- assert not_found_result == None
-
- m.get(
- url=f"{state_sdk.wb.host}/v1/drifting-metrics?model_id={mock_model_id}",
- headers={"api-key": state_sdk.wb.api_key},
- json=drifting_metrics_report_payload,
- )
-
- drifting_report = state_sdk.wb.get_drifting_metrics(model_id=mock_model_id)
-
- assert drifting_report == drifting_metrics_report_payload
-
-
-@pytest.mark.order(get_order_number("sdk_get_descriptive_statistics"))
-def test_sdk_get_descriptive_statistics(client):
- mock_model_id = "mock_model_id"
- with requests_mock.Mocker() as m:
- m.get(
- url=f"{state_sdk.wb.host}/v1/model-integrity-metrics?model_id={mock_model_id}",
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_404_NOT_FOUND,
- )
-
- not_found_result = state_sdk.wb.get_descriptive_statistics(
- model_id=mock_model_id
- )
-
- assert not_found_result == None
-
- m.get(
- url=f"{state_sdk.wb.host}/v1/model-integrity-metrics?model_id={mock_model_id}",
- headers={"api-key": state_sdk.wb.api_key},
- json=descriptive_statistics_report_payload,
- )
-
- descriptive_report = state_sdk.wb.get_descriptive_statistics(
- model_id=mock_model_id
- )
-
- assert descriptive_report == descriptive_statistics_report_payload
-
-
-@pytest.mark.order(get_order_number("sdk_get_performance_metrics"))
-def test_sdk_get_performance_metrics(client):
- mock_model_id = "mock_model_id"
- with requests_mock.Mocker() as m:
- m.get(
- url=f"{state_sdk.wb.host}/v1/performance-metrics?model_id={mock_model_id}",
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_404_NOT_FOUND,
- )
-
- not_found_result = state_sdk.wb.get_performance_metrics(model_id=mock_model_id)
-
- assert not_found_result == None
-
- m.get(
- url=f"{state_sdk.wb.host}/v1/performance-metrics?model_id={mock_model_id}",
- headers={"api-key": state_sdk.wb.api_key},
- json=performance_metrics_report_payload,
- )
-
- performance_report = state_sdk.wb.get_performance_metrics(
- model_id=mock_model_id
- )
-
- assert performance_report == performance_metrics_report_payload
-
-
-@pytest.mark.order(get_order_number("sdk_get_xai_row"))
-def test_sdk_get_xai_row(client):
- mock_inference_id = "mock_inference_id"
- with requests_mock.Mocker() as m:
- m.get(
- url=f"{state_sdk.wb.host}/v1/inference-rows/{mock_inference_id}/xai",
- headers={"api-key": state_sdk.wb.api_key},
- status_code=status.HTTP_404_NOT_FOUND,
- )
-
- not_found_result = state_sdk.wb.get_xai_row(inference_row_id=mock_inference_id)
-
- assert not_found_result == None
-
- m.get(
- url=f"{state_sdk.wb.host}/v1/inference-rows/{mock_inference_id}/xai",
- headers={"api-key": state_sdk.wb.api_key},
- json=inference_row_xai_payload,
- )
-
- xai = state_sdk.wb.get_xai_row(inference_row_id=mock_inference_id)
-
- assert xai == inference_row_xai_payload
diff --git a/whitebox/utils/__init__.py b/whitebox/utils/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/whitebox/utils/errors.py b/whitebox/utils/errors.py
deleted file mode 100644
index 6b7c050..0000000
--- a/whitebox/utils/errors.py
+++ /dev/null
@@ -1,124 +0,0 @@
-from typing import List
-from fastapi import status
-from fastapi.encoders import jsonable_encoder
-from fastapi.responses import JSONResponse
-from starlette.exceptions import HTTPException as StarletteHTTPException
-from fastapi.exceptions import HTTPException
-from starlette.requests import Request
-from whitebox.schemas.utils import ErrorProps
-from whitebox.utils.logger import log
-
-
-class CustomError(BaseException):
- async def http_exception_handler(
- self, _: Request, exc: StarletteHTTPException
- ) -> ErrorProps:
- log.error(f"{exc.status_code}: {exc.detail}")
- return JSONResponse(
- status_code=exc.status_code,
- content=jsonable_encoder(
- {"error": f"{str(exc.detail)}", "status_code": exc.status_code}
- ),
- )
-
- async def validation_exception_handler(
- self,
- _: Request,
- exc: HTTPException,
- ) -> ErrorProps:
- responsible_value = exc.errors()[0]["loc"][-1]
- reason = exc.errors()[0]["msg"]
- log.error(
- f"{status.HTTP_422_UNPROCESSABLE_ENTITY}: ({str(responsible_value)}) {str(reason)}"
- )
- return JSONResponse(
- status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
- content=jsonable_encoder(
- {
- "error": f"({str(responsible_value)}) {str(reason)}",
- "status_code": status.HTTP_422_UNPROCESSABLE_ENTITY,
- }
- ),
- )
-
- def bad_request(self, msg: str = "Bad request") -> ErrorProps:
- log.error(f"{status.HTTP_400_BAD_REQUEST}: {str(msg)}")
- return JSONResponse(
- status_code=status.HTTP_400_BAD_REQUEST,
- content=jsonable_encoder(
- {"error": str(msg), "status_code": status.HTTP_400_BAD_REQUEST}
- ),
- )
-
- def not_found(self, msg: str = "Content not found") -> ErrorProps:
- log.error(f"{status.HTTP_404_NOT_FOUND}: {str(msg)}")
- return JSONResponse(
- status_code=status.HTTP_404_NOT_FOUND,
- content=jsonable_encoder(
- {"error": str(msg), "status_code": status.HTTP_404_NOT_FOUND}
- ),
- )
-
-
-errors = CustomError()
-
-
-def add_error_responses(status_codes) -> List[ErrorProps]:
- """
- For the schema to work the part after schemas/ should correspond to a title error schema in whitebox/api/app/v1/docs.py
- """
-
- error_responses = {
- 400: {
- "description": "Bad Request",
- "content": {
- "application/json": {
- "schema": {"$ref": "#/components/schemas/BadRequest"}
- }
- },
- },
- 401: {
- "description": "Authorization Error",
- "content": {
- "application/json": {
- "schema": {"$ref": "#/components/schemas/AuthorizationError"}
- }
- },
- },
- 404: {
- "description": "Not Found Error",
- "content": {
- "application/json": {
- "schema": {"$ref": "#/components/schemas/NotFoundError"}
- }
- },
- },
- 409: {
- "description": "Conflict Error",
- "content": {
- "application/json": {
- "schema": {"$ref": "#/components/schemas/ConflictError"}
- }
- },
- },
- 410: {
- "description": "Content Gone",
- "content": {
- "application/json": {
- "schema": {"$ref": "#/components/schemas/ContenGone"}
- }
- },
- },
- 422: {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {"$ref": "#/components/schemas/HTTPValidationError"}
- }
- },
- },
- }
- responses = {}
- for code in status_codes:
- responses[code] = error_responses[code]
- return responses
diff --git a/whitebox/utils/exceptions.py b/whitebox/utils/exceptions.py
deleted file mode 100644
index 8268fa0..0000000
--- a/whitebox/utils/exceptions.py
+++ /dev/null
@@ -1,10 +0,0 @@
-class TaskAlreadyRunningException(Exception):
- pass
-
-
-class TaskNotFoundException(Exception):
- pass
-
-
-class TaskNotRunningException(Exception):
- pass
diff --git a/whitebox/utils/id_gen.py b/whitebox/utils/id_gen.py
deleted file mode 100644
index 9ec92e6..0000000
--- a/whitebox/utils/id_gen.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from uuid import uuid4
-
-
-def generate_uuid():
- return str(uuid4())
diff --git a/whitebox/utils/logger.py b/whitebox/utils/logger.py
deleted file mode 100644
index 8326174..0000000
--- a/whitebox/utils/logger.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from colorama import Fore, Back, Style
-import logging
-
-
-class Logger:
- def info(self, info):
- print(Fore.LIGHTBLUE_EX + "INFO" + Fore.BLACK + ":" + 4 * " ", info)
-
- def error(self, error):
- print(Fore.RED + "ERROR" + Fore.BLACK + ":" + 3 * " ", error)
-
- def success(self, msg):
- print(Fore.GREEN + "SUCCESS" + Fore.BLACK + ":" + 1 * " ", msg)
-
-
-log = Logger()
-
-
-logging.basicConfig(level="INFO")
-
-cronLogger = logging.getLogger("cron")
diff --git a/whitebox/utils/passwords.py b/whitebox/utils/passwords.py
deleted file mode 100644
index 3386dd2..0000000
--- a/whitebox/utils/passwords.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
-from cryptography.hazmat.primitives import padding
-from cryptography.hazmat.backends import default_backend
-import secrets
-from whitebox.core.settings import get_settings
-
-
-settings = get_settings()
-
-
-def to_utf8(ps):
- return ps.encode("utf-8")
-
-
-def encrypt_api_key(password: str, key: bytes) -> str:
- iv = secrets.token_bytes(16)
- cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
- encryptor = cipher.encryptor()
- padder = padding.PKCS7(128).padder()
- padded_data = padder.update(to_utf8(password)) + padder.finalize()
- ct = encryptor.update(padded_data) + encryptor.finalize()
- return (iv + ct).hex()
-
-
-def decrypt_api_key(password: str, key: bytes) -> str:
- ct = bytes.fromhex(password)
- iv = ct[:16]
- ct = ct[16:]
- cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
- decryptor = cipher.decryptor()
- pt = decryptor.update(ct) + decryptor.finalize()
- unpadder = padding.PKCS7(128).unpadder()
- data = unpadder.update(pt) + unpadder.finalize()
- return data.decode("utf-8")
-
-
-def passwords_match(api_key_in_db: str, api_key_in_headers: str):
- if not settings.SECRET_KEY:
- return api_key_in_db == api_key_in_headers
-
- api_key = decrypt_api_key(api_key_in_db, settings.SECRET_KEY.encode())
- return api_key == api_key_in_headers
+
+