[{"data":1,"prerenderedAt":1618},["ShallowReactive",2],{"content-query-6rKhLm2VAr":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"body":10,"_type":1612,"_id":1613,"_source":1614,"_file":1615,"_stem":1616,"_extension":1617},"\u002Fdocs\u002Ffeatures\u002Fdata-workflow","features",false,"","Data Workflow Guide","Complete workflow for data analysis, experimentation, and production",{"type":11,"children":12,"toc":1572},"root",[13,21,27,34,155,161,170,195,202,210,216,383,393,399,407,430,436,444,450,614,622,630,639,645,652,675,681,689,695,703,711,1103,1112,1118,1126,1149,1155,1163,1169,1177,1185,1193,1199,1207,1235,1241,1249,1255,1261,1269,1275,1283,1289,1297,1303,1311,1317,1323,1331,1337,1345,1351,1359,1365,1371,1399,1405,1433,1439,1467,1473,1501,1507,1566],{"type":14,"tag":15,"props":16,"children":18},"element","h1",{"id":17},"data-workflow-guide",[19],{"type":20,"value":8},"text",{"type":14,"tag":22,"props":23,"children":24},"p",{},[25],{"type":20,"value":26},"Learn how Notebook, ML Experiments, Orchestration, Lineage, and Metadata work together as a complete data platform.",{"type":14,"tag":28,"props":29,"children":31},"h2",{"id":30},"the-data-platform-workflow",[32],{"type":20,"value":33},"The Data Platform Workflow",{"type":14,"tag":35,"props":36,"children":40},"pre",{"code":37,"language":38,"meta":7,"className":39,"style":7},"graph TD\n    A[\"EXPLORE\u003Cbr\u002F>━━━━━\u003Cbr\u002F>Notebook\u003Cbr\u002F>Load & Analyze\"] --> B[\"EXPERIMENT\u003Cbr\u002F>━━━━━━\u003Cbr\u002F>ML Experiments\u003Cbr\u002F>Train Models\"]\n    B --> C[\"AUTOMATE\u003Cbr\u002F>━━━━━━\u003Cbr\u002F>Orchestration\u003Cbr\u002F>Run Pipelines\"]\n    C --> D[\"TRACK\u003Cbr\u002F>━━━━\u003Cbr\u002F>Lineage + Metadata\u003Cbr\u002F>Monitor Flow\"]\n    D --> E[\"SHARE\u003Cbr\u002F>━━━━\u003Cbr\u002F>Documentation\u003Cbr\u002F>Collaborate\"]\n    E -.-> A\n    \n    style A fill:#1f97d4,stroke:#0b5394,stroke-width:2px,color:#fff\n    style B fill:#ff9900,stroke:#ec7211,stroke-width:2px,color:#fff\n    style C fill:#37475a,stroke:#1f1f1f,stroke-width:2px,color:#fff\n    style D fill:#1f97d4,stroke:#0b5394,stroke-width:2px,color:#fff\n    style E fill:#ff9900,stroke:#ec7211,stroke-width:2px,color:#fff\n","mermaid","language-mermaid shiki shiki-themes github-dark",[41],{"type":14,"tag":42,"props":43,"children":44},"code",{"__ignoreMap":7},[45,56,65,74,83,92,101,110,119,128,137,146],{"type":14,"tag":46,"props":47,"children":50},"span",{"class":48,"line":49},"line",1,[51],{"type":14,"tag":46,"props":52,"children":53},{},[54],{"type":20,"value":55},"graph TD\n",{"type":14,"tag":46,"props":57,"children":59},{"class":48,"line":58},2,[60],{"type":14,"tag":46,"props":61,"children":62},{},[63],{"type":20,"value":64},"    A[\"EXPLORE\u003Cbr\u002F>━━━━━\u003Cbr\u002F>Notebook\u003Cbr\u002F>Load & Analyze\"] --> B[\"EXPERIMENT\u003Cbr\u002F>━━━━━━\u003Cbr\u002F>ML Experiments\u003Cbr\u002F>Train Models\"]\n",{"type":14,"tag":46,"props":66,"children":68},{"class":48,"line":67},3,[69],{"type":14,"tag":46,"props":70,"children":71},{},[72],{"type":20,"value":73},"    B --> C[\"AUTOMATE\u003Cbr\u002F>━━━━━━\u003Cbr\u002F>Orchestration\u003Cbr\u002F>Run Pipelines\"]\n",{"type":14,"tag":46,"props":75,"children":77},{"class":48,"line":76},4,[78],{"type":14,"tag":46,"props":79,"children":80},{},[81],{"type":20,"value":82},"    C --> D[\"TRACK\u003Cbr\u002F>━━━━\u003Cbr\u002F>Lineage + Metadata\u003Cbr\u002F>Monitor Flow\"]\n",{"type":14,"tag":46,"props":84,"children":86},{"class":48,"line":85},5,[87],{"type":14,"tag":46,"props":88,"children":89},{},[90],{"type":20,"value":91},"    D --> E[\"SHARE\u003Cbr\u002F>━━━━\u003Cbr\u002F>Documentation\u003Cbr\u002F>Collaborate\"]\n",{"type":14,"tag":46,"props":93,"children":95},{"class":48,"line":94},6,[96],{"type":14,"tag":46,"props":97,"children":98},{},[99],{"type":20,"value":100},"    E -.-> A\n",{"type":14,"tag":46,"props":102,"children":104},{"class":48,"line":103},7,[105],{"type":14,"tag":46,"props":106,"children":107},{},[108],{"type":20,"value":109},"    \n",{"type":14,"tag":46,"props":111,"children":113},{"class":48,"line":112},8,[114],{"type":14,"tag":46,"props":115,"children":116},{},[117],{"type":20,"value":118},"    style A fill:#1f97d4,stroke:#0b5394,stroke-width:2px,color:#fff\n",{"type":14,"tag":46,"props":120,"children":122},{"class":48,"line":121},9,[123],{"type":14,"tag":46,"props":124,"children":125},{},[126],{"type":20,"value":127},"    style B fill:#ff9900,stroke:#ec7211,stroke-width:2px,color:#fff\n",{"type":14,"tag":46,"props":129,"children":131},{"class":48,"line":130},10,[132],{"type":14,"tag":46,"props":133,"children":134},{},[135],{"type":20,"value":136},"    style C fill:#37475a,stroke:#1f1f1f,stroke-width:2px,color:#fff\n",{"type":14,"tag":46,"props":138,"children":140},{"class":48,"line":139},11,[141],{"type":14,"tag":46,"props":142,"children":143},{},[144],{"type":20,"value":145},"    style D fill:#1f97d4,stroke:#0b5394,stroke-width:2px,color:#fff\n",{"type":14,"tag":46,"props":147,"children":149},{"class":48,"line":148},12,[150],{"type":14,"tag":46,"props":151,"children":152},{},[153],{"type":20,"value":154},"    style E fill:#ff9900,stroke:#ec7211,stroke-width:2px,color:#fff\n",{"type":14,"tag":28,"props":156,"children":158},{"id":157},"phase-1-explore-data-in-notebook",[159],{"type":20,"value":160},"Phase 1: Explore Data in Notebook",{"type":14,"tag":22,"props":162,"children":163},{},[164],{"type":14,"tag":165,"props":166,"children":167},"strong",{},[168],{"type":20,"value":169},"Start here when:",{"type":14,"tag":171,"props":172,"children":173},"ul",{},[174,180,185,190],{"type":14,"tag":175,"props":176,"children":177},"li",{},[178],{"type":20,"value":179},"You have a new dataset",{"type":14,"tag":175,"props":181,"children":182},{},[183],{"type":20,"value":184},"Want to understand patterns",{"type":14,"tag":175,"props":186,"children":187},{},[188],{"type":20,"value":189},"Testing hypotheses",{"type":14,"tag":175,"props":191,"children":192},{},[193],{"type":20,"value":194},"Prototyping analysis",{"type":14,"tag":196,"props":197,"children":199},"h3",{"id":198},"notebook-is-your-lab",[200],{"type":20,"value":201},"Notebook Is Your Lab",{"type":14,"tag":35,"props":203,"children":205},{"code":204},"What you do:\n✓ Load data from databases\n✓ Write analysis code\n✓ Visualize findings\n✓ Test approaches\n✓ Iterate quickly\n\nTools:\n- Python\u002FSQL cells\n- Instant feedback\n- Live visualizations\n- Easy data exploration\n",[206],{"type":14,"tag":42,"props":207,"children":208},{"__ignoreMap":7},[209],{"type":20,"value":204},{"type":14,"tag":196,"props":211,"children":213},{"id":212},"example-customer-analysis",[214],{"type":20,"value":215},"Example: Customer Analysis",{"type":14,"tag":35,"props":217,"children":221},{"code":218,"language":219,"meta":7,"className":220,"style":7},"# Cell 1: Load customer data\nimport pandas as pd\ncustomers = pd.read_sql(\"\"\"\n  SELECT * FROM customers \n  WHERE created_at > '2024-01-01'\n\"\"\", connection)\n\n# Cell 2: Explore\nprint(f\"Total customers: {len(customers)}\")\nprint(customers.head())\n\n# Cell 3: Analyze\nchurn_rate = (customers[customers['status'] == 'inactive'].shape[0] \u002F len(customers)) * 100\nprint(f\"Churn rate: {churn_rate}%\")\n\n# Cell 4: Visualize\nimport matplotlib.pyplot as plt\ncustomers.groupby('region')['revenue'].sum().plot(kind='bar')\nplt.show()\n","python","language-python shiki shiki-themes github-dark",[222],{"type":14,"tag":42,"props":223,"children":224},{"__ignoreMap":7},[225,233,241,249,257,265,273,282,290,298,306,313,321,330,339,347,356,365,374],{"type":14,"tag":46,"props":226,"children":227},{"class":48,"line":49},[228],{"type":14,"tag":46,"props":229,"children":230},{},[231],{"type":20,"value":232},"# Cell 1: Load customer data\n",{"type":14,"tag":46,"props":234,"children":235},{"class":48,"line":58},[236],{"type":14,"tag":46,"props":237,"children":238},{},[239],{"type":20,"value":240},"import pandas as pd\n",{"type":14,"tag":46,"props":242,"children":243},{"class":48,"line":67},[244],{"type":14,"tag":46,"props":245,"children":246},{},[247],{"type":20,"value":248},"customers = pd.read_sql(\"\"\"\n",{"type":14,"tag":46,"props":250,"children":251},{"class":48,"line":76},[252],{"type":14,"tag":46,"props":253,"children":254},{},[255],{"type":20,"value":256},"  SELECT * FROM customers \n",{"type":14,"tag":46,"props":258,"children":259},{"class":48,"line":85},[260],{"type":14,"tag":46,"props":261,"children":262},{},[263],{"type":20,"value":264},"  WHERE created_at > '2024-01-01'\n",{"type":14,"tag":46,"props":266,"children":267},{"class":48,"line":94},[268],{"type":14,"tag":46,"props":269,"children":270},{},[271],{"type":20,"value":272},"\"\"\", connection)\n",{"type":14,"tag":46,"props":274,"children":275},{"class":48,"line":103},[276],{"type":14,"tag":46,"props":277,"children":279},{"emptyLinePlaceholder":278},true,[280],{"type":20,"value":281},"\n",{"type":14,"tag":46,"props":283,"children":284},{"class":48,"line":112},[285],{"type":14,"tag":46,"props":286,"children":287},{},[288],{"type":20,"value":289},"# Cell 2: Explore\n",{"type":14,"tag":46,"props":291,"children":292},{"class":48,"line":121},[293],{"type":14,"tag":46,"props":294,"children":295},{},[296],{"type":20,"value":297},"print(f\"Total customers: {len(customers)}\")\n",{"type":14,"tag":46,"props":299,"children":300},{"class":48,"line":130},[301],{"type":14,"tag":46,"props":302,"children":303},{},[304],{"type":20,"value":305},"print(customers.head())\n",{"type":14,"tag":46,"props":307,"children":308},{"class":48,"line":139},[309],{"type":14,"tag":46,"props":310,"children":311},{"emptyLinePlaceholder":278},[312],{"type":20,"value":281},{"type":14,"tag":46,"props":314,"children":315},{"class":48,"line":148},[316],{"type":14,"tag":46,"props":317,"children":318},{},[319],{"type":20,"value":320},"# Cell 3: Analyze\n",{"type":14,"tag":46,"props":322,"children":324},{"class":48,"line":323},13,[325],{"type":14,"tag":46,"props":326,"children":327},{},[328],{"type":20,"value":329},"churn_rate = (customers[customers['status'] == 'inactive'].shape[0] \u002F len(customers)) * 100\n",{"type":14,"tag":46,"props":331,"children":333},{"class":48,"line":332},14,[334],{"type":14,"tag":46,"props":335,"children":336},{},[337],{"type":20,"value":338},"print(f\"Churn rate: {churn_rate}%\")\n",{"type":14,"tag":46,"props":340,"children":342},{"class":48,"line":341},15,[343],{"type":14,"tag":46,"props":344,"children":345},{"emptyLinePlaceholder":278},[346],{"type":20,"value":281},{"type":14,"tag":46,"props":348,"children":350},{"class":48,"line":349},16,[351],{"type":14,"tag":46,"props":352,"children":353},{},[354],{"type":20,"value":355},"# Cell 4: Visualize\n",{"type":14,"tag":46,"props":357,"children":359},{"class":48,"line":358},17,[360],{"type":14,"tag":46,"props":361,"children":362},{},[363],{"type":20,"value":364},"import matplotlib.pyplot as plt\n",{"type":14,"tag":46,"props":366,"children":368},{"class":48,"line":367},18,[369],{"type":14,"tag":46,"props":370,"children":371},{},[372],{"type":20,"value":373},"customers.groupby('region')['revenue'].sum().plot(kind='bar')\n",{"type":14,"tag":46,"props":375,"children":377},{"class":48,"line":376},19,[378],{"type":14,"tag":46,"props":379,"children":380},{},[381],{"type":20,"value":382},"plt.show()\n",{"type":14,"tag":22,"props":384,"children":385},{},[386,391],{"type":14,"tag":165,"props":387,"children":388},{},[389],{"type":20,"value":390},"Next step:",{"type":20,"value":392}," If analysis works and is valuable → Move to ML Experiments",{"type":14,"tag":28,"props":394,"children":396},{"id":395},"phase-2-track-experiments",[397],{"type":20,"value":398},"Phase 2: Track Experiments",{"type":14,"tag":22,"props":400,"children":401},{},[402],{"type":14,"tag":165,"props":403,"children":404},{},[405],{"type":20,"value":406},"Move here when:",{"type":14,"tag":171,"props":408,"children":409},{},[410,415,420,425],{"type":14,"tag":175,"props":411,"children":412},{},[413],{"type":20,"value":414},"Analysis is solid",{"type":14,"tag":175,"props":416,"children":417},{},[418],{"type":20,"value":419},"Want to try variations",{"type":14,"tag":175,"props":421,"children":422},{},[423],{"type":20,"value":424},"Need to compare results",{"type":14,"tag":175,"props":426,"children":427},{},[428],{"type":20,"value":429},"Building a model",{"type":14,"tag":196,"props":431,"children":433},{"id":432},"ml-experiments-track-your-progress",[434],{"type":20,"value":435},"ML Experiments Track Your Progress",{"type":14,"tag":35,"props":437,"children":439},{"code":438},"What you do:\n✓ Try multiple approaches\n✓ Log metrics from each run\n✓ Compare performance\n✓ Keep best version\n✓ Document what worked\n\nTools:\n- Parameter logging\n- Metric tracking\n- Automatic comparison\n- Model versioning\n",[440],{"type":14,"tag":42,"props":441,"children":442},{"__ignoreMap":7},[443],{"type":20,"value":438},{"type":14,"tag":196,"props":445,"children":447},{"id":446},"example-building-customer-churn-model",[448],{"type":20,"value":449},"Example: Building Customer Churn Model",{"type":14,"tag":35,"props":451,"children":453},{"code":452,"language":219,"meta":7,"className":220,"style":7},"from credvault import experiment\n\n# Start experiment\nexp = experiment.start('Churn Prediction')\n\n# Try approach 1: Simple logistic regression\nexp.log_params({'model': 'logistic_regression', 'features': 10})\nmodel1 = train_logistic_regression(data)\naccuracy1 = evaluate(model1)\nexp.log_metric('accuracy', accuracy1)\n\n# Try approach 2: Random forest with more features\nexp.log_params({'model': 'random_forest', 'features': 25})\nmodel2 = train_random_forest(data)\naccuracy2 = evaluate(model2)\nexp.log_metric('accuracy', accuracy2)\n\n# Compare in ML Experiments dashboard\n# Decide: Random forest (92% accuracy) > Logistic (87%)\nexp.save_as_model('v1.0-production')\n",[454],{"type":14,"tag":42,"props":455,"children":456},{"__ignoreMap":7},[457,465,472,480,488,495,503,511,519,527,535,542,550,558,566,574,582,589,597,605],{"type":14,"tag":46,"props":458,"children":459},{"class":48,"line":49},[460],{"type":14,"tag":46,"props":461,"children":462},{},[463],{"type":20,"value":464},"from credvault import experiment\n",{"type":14,"tag":46,"props":466,"children":467},{"class":48,"line":58},[468],{"type":14,"tag":46,"props":469,"children":470},{"emptyLinePlaceholder":278},[471],{"type":20,"value":281},{"type":14,"tag":46,"props":473,"children":474},{"class":48,"line":67},[475],{"type":14,"tag":46,"props":476,"children":477},{},[478],{"type":20,"value":479},"# Start experiment\n",{"type":14,"tag":46,"props":481,"children":482},{"class":48,"line":76},[483],{"type":14,"tag":46,"props":484,"children":485},{},[486],{"type":20,"value":487},"exp = experiment.start('Churn Prediction')\n",{"type":14,"tag":46,"props":489,"children":490},{"class":48,"line":85},[491],{"type":14,"tag":46,"props":492,"children":493},{"emptyLinePlaceholder":278},[494],{"type":20,"value":281},{"type":14,"tag":46,"props":496,"children":497},{"class":48,"line":94},[498],{"type":14,"tag":46,"props":499,"children":500},{},[501],{"type":20,"value":502},"# Try approach 1: Simple logistic regression\n",{"type":14,"tag":46,"props":504,"children":505},{"class":48,"line":103},[506],{"type":14,"tag":46,"props":507,"children":508},{},[509],{"type":20,"value":510},"exp.log_params({'model': 'logistic_regression', 'features': 10})\n",{"type":14,"tag":46,"props":512,"children":513},{"class":48,"line":112},[514],{"type":14,"tag":46,"props":515,"children":516},{},[517],{"type":20,"value":518},"model1 = train_logistic_regression(data)\n",{"type":14,"tag":46,"props":520,"children":521},{"class":48,"line":121},[522],{"type":14,"tag":46,"props":523,"children":524},{},[525],{"type":20,"value":526},"accuracy1 = evaluate(model1)\n",{"type":14,"tag":46,"props":528,"children":529},{"class":48,"line":130},[530],{"type":14,"tag":46,"props":531,"children":532},{},[533],{"type":20,"value":534},"exp.log_metric('accuracy', accuracy1)\n",{"type":14,"tag":46,"props":536,"children":537},{"class":48,"line":139},[538],{"type":14,"tag":46,"props":539,"children":540},{"emptyLinePlaceholder":278},[541],{"type":20,"value":281},{"type":14,"tag":46,"props":543,"children":544},{"class":48,"line":148},[545],{"type":14,"tag":46,"props":546,"children":547},{},[548],{"type":20,"value":549},"# Try approach 2: Random forest with more features\n",{"type":14,"tag":46,"props":551,"children":552},{"class":48,"line":323},[553],{"type":14,"tag":46,"props":554,"children":555},{},[556],{"type":20,"value":557},"exp.log_params({'model': 'random_forest', 'features': 25})\n",{"type":14,"tag":46,"props":559,"children":560},{"class":48,"line":332},[561],{"type":14,"tag":46,"props":562,"children":563},{},[564],{"type":20,"value":565},"model2 = train_random_forest(data)\n",{"type":14,"tag":46,"props":567,"children":568},{"class":48,"line":341},[569],{"type":14,"tag":46,"props":570,"children":571},{},[572],{"type":20,"value":573},"accuracy2 = evaluate(model2)\n",{"type":14,"tag":46,"props":575,"children":576},{"class":48,"line":349},[577],{"type":14,"tag":46,"props":578,"children":579},{},[580],{"type":20,"value":581},"exp.log_metric('accuracy', accuracy2)\n",{"type":14,"tag":46,"props":583,"children":584},{"class":48,"line":358},[585],{"type":14,"tag":46,"props":586,"children":587},{"emptyLinePlaceholder":278},[588],{"type":20,"value":281},{"type":14,"tag":46,"props":590,"children":591},{"class":48,"line":367},[592],{"type":14,"tag":46,"props":593,"children":594},{},[595],{"type":20,"value":596},"# Compare in ML Experiments dashboard\n",{"type":14,"tag":46,"props":598,"children":599},{"class":48,"line":376},[600],{"type":14,"tag":46,"props":601,"children":602},{},[603],{"type":20,"value":604},"# Decide: Random forest (92% accuracy) > Logistic (87%)\n",{"type":14,"tag":46,"props":606,"children":608},{"class":48,"line":607},20,[609],{"type":14,"tag":46,"props":610,"children":611},{},[612],{"type":20,"value":613},"exp.save_as_model('v1.0-production')\n",{"type":14,"tag":22,"props":615,"children":616},{},[617],{"type":14,"tag":165,"props":618,"children":619},{},[620],{"type":20,"value":621},"Results in ML Experiments:",{"type":14,"tag":35,"props":623,"children":625},{"code":624},"Run 1: Accuracy 87% ← Logistic Regression\nRun 2: Accuracy 92% ← Random Forest (BEST)\n",[626],{"type":14,"tag":42,"props":627,"children":628},{"__ignoreMap":7},[629],{"type":20,"value":624},{"type":14,"tag":22,"props":631,"children":632},{},[633,637],{"type":14,"tag":165,"props":634,"children":635},{},[636],{"type":20,"value":390},{"type":20,"value":638}," Model works well → Automate it with Orchestration",{"type":14,"tag":28,"props":640,"children":642},{"id":641},"phase-3-automate-with-orchestration",[643],{"type":20,"value":644},"Phase 3: Automate with Orchestration",{"type":14,"tag":22,"props":646,"children":647},{},[648],{"type":14,"tag":165,"props":649,"children":650},{},[651],{"type":20,"value":406},{"type":14,"tag":171,"props":653,"children":654},{},[655,660,665,670],{"type":14,"tag":175,"props":656,"children":657},{},[658],{"type":20,"value":659},"Model\u002Fanalysis is finalized",{"type":14,"tag":175,"props":661,"children":662},{},[663],{"type":20,"value":664},"Need to run regularly",{"type":14,"tag":175,"props":666,"children":667},{},[668],{"type":20,"value":669},"Process many datasets",{"type":14,"tag":175,"props":671,"children":672},{},[673],{"type":20,"value":674},"Multiple steps involved",{"type":14,"tag":196,"props":676,"children":678},{"id":677},"orchestration-runs-on-schedule",[679],{"type":20,"value":680},"Orchestration Runs on Schedule",{"type":14,"tag":35,"props":682,"children":684},{"code":683},"What you do:\n✓ Define workflow steps\n✓ Schedule daily\u002Fhourly\n✓ Handle dependencies\n✓ Manage retries\n✓ Monitor execution\n\nPipeline:\n1. Extract data from databases\n2. Transform and validate\n3. Run trained model\n4. Load results\n5. Generate reports\n",[685],{"type":14,"tag":42,"props":686,"children":687},{"__ignoreMap":7},[688],{"type":20,"value":683},{"type":14,"tag":196,"props":690,"children":692},{"id":691},"example-daily-churn-prediction-pipeline",[693],{"type":20,"value":694},"Example: Daily Churn Prediction Pipeline",{"type":14,"tag":35,"props":696,"children":698},{"code":697},"9:00 AM:\n  ↓\nStep 1: Extract fresh customer data (5 min)\n  ↓\nStep 2: Validate data quality (2 min)\n  ↓\nStep 3: Run churn model (3 min)\n  ↓\nStep 4: Load predictions to database (1 min)\n  ↓\nStep 5: Generate report & send to team (1 min)\n  ↓\n9:12 AM: Done! Results ready\n",[699],{"type":14,"tag":42,"props":700,"children":701},{"__ignoreMap":7},[702],{"type":20,"value":697},{"type":14,"tag":22,"props":704,"children":705},{},[706],{"type":14,"tag":165,"props":707,"children":708},{},[709],{"type":20,"value":710},"In Orchestration:",{"type":14,"tag":35,"props":712,"children":716},{"code":713,"language":714,"meta":7,"className":715,"style":7},"name: Daily Churn Prediction\n\nschedule: \"0 9 * * *\"  # 9 AM every day\n\nsteps:\n  - name: extract\n    query: SELECT * FROM customers WHERE active = true\n    \n  - name: validate\n    checks:\n      - no_nulls: email\n      - unique: customer_id\n    \n  - name: predict\n    model: churn_v1.0\n    input: extract.output\n    \n  - name: load\n    destination: predictions_daily\n    input: predict.output\n    \n  - name: report\n    template: churn_daily_report\n    recipients: [team@company.com]\n","yaml","language-yaml shiki shiki-themes github-dark",[717],{"type":14,"tag":42,"props":718,"children":719},{"__ignoreMap":7},[720,741,748,771,778,791,812,829,836,856,868,890,911,918,938,955,972,979,999,1016,1032,1040,1061,1079],{"type":14,"tag":46,"props":721,"children":722},{"class":48,"line":49},[723,729,735],{"type":14,"tag":46,"props":724,"children":726},{"style":725},"--shiki-default:#85E89D",[727],{"type":20,"value":728},"name",{"type":14,"tag":46,"props":730,"children":732},{"style":731},"--shiki-default:#E1E4E8",[733],{"type":20,"value":734},": ",{"type":14,"tag":46,"props":736,"children":738},{"style":737},"--shiki-default:#9ECBFF",[739],{"type":20,"value":740},"Daily Churn Prediction\n",{"type":14,"tag":46,"props":742,"children":743},{"class":48,"line":58},[744],{"type":14,"tag":46,"props":745,"children":746},{"emptyLinePlaceholder":278},[747],{"type":20,"value":281},{"type":14,"tag":46,"props":749,"children":750},{"class":48,"line":67},[751,756,760,765],{"type":14,"tag":46,"props":752,"children":753},{"style":725},[754],{"type":20,"value":755},"schedule",{"type":14,"tag":46,"props":757,"children":758},{"style":731},[759],{"type":20,"value":734},{"type":14,"tag":46,"props":761,"children":762},{"style":737},[763],{"type":20,"value":764},"\"0 9 * * *\"",{"type":14,"tag":46,"props":766,"children":768},{"style":767},"--shiki-default:#6A737D",[769],{"type":20,"value":770},"  # 9 AM every day\n",{"type":14,"tag":46,"props":772,"children":773},{"class":48,"line":76},[774],{"type":14,"tag":46,"props":775,"children":776},{"emptyLinePlaceholder":278},[777],{"type":20,"value":281},{"type":14,"tag":46,"props":779,"children":780},{"class":48,"line":85},[781,786],{"type":14,"tag":46,"props":782,"children":783},{"style":725},[784],{"type":20,"value":785},"steps",{"type":14,"tag":46,"props":787,"children":788},{"style":731},[789],{"type":20,"value":790},":\n",{"type":14,"tag":46,"props":792,"children":793},{"class":48,"line":94},[794,799,803,807],{"type":14,"tag":46,"props":795,"children":796},{"style":731},[797],{"type":20,"value":798},"  - ",{"type":14,"tag":46,"props":800,"children":801},{"style":725},[802],{"type":20,"value":728},{"type":14,"tag":46,"props":804,"children":805},{"style":731},[806],{"type":20,"value":734},{"type":14,"tag":46,"props":808,"children":809},{"style":737},[810],{"type":20,"value":811},"extract\n",{"type":14,"tag":46,"props":813,"children":814},{"class":48,"line":103},[815,820,824],{"type":14,"tag":46,"props":816,"children":817},{"style":725},[818],{"type":20,"value":819},"    query",{"type":14,"tag":46,"props":821,"children":822},{"style":731},[823],{"type":20,"value":734},{"type":14,"tag":46,"props":825,"children":826},{"style":737},[827],{"type":20,"value":828},"SELECT * FROM customers WHERE active = true\n",{"type":14,"tag":46,"props":830,"children":831},{"class":48,"line":112},[832],{"type":14,"tag":46,"props":833,"children":834},{"style":731},[835],{"type":20,"value":109},{"type":14,"tag":46,"props":837,"children":838},{"class":48,"line":121},[839,843,847,851],{"type":14,"tag":46,"props":840,"children":841},{"style":731},[842],{"type":20,"value":798},{"type":14,"tag":46,"props":844,"children":845},{"style":725},[846],{"type":20,"value":728},{"type":14,"tag":46,"props":848,"children":849},{"style":731},[850],{"type":20,"value":734},{"type":14,"tag":46,"props":852,"children":853},{"style":737},[854],{"type":20,"value":855},"validate\n",{"type":14,"tag":46,"props":857,"children":858},{"class":48,"line":130},[859,864],{"type":14,"tag":46,"props":860,"children":861},{"style":725},[862],{"type":20,"value":863},"    checks",{"type":14,"tag":46,"props":865,"children":866},{"style":731},[867],{"type":20,"value":790},{"type":14,"tag":46,"props":869,"children":870},{"class":48,"line":139},[871,876,881,885],{"type":14,"tag":46,"props":872,"children":873},{"style":731},[874],{"type":20,"value":875},"      - ",{"type":14,"tag":46,"props":877,"children":878},{"style":725},[879],{"type":20,"value":880},"no_nulls",{"type":14,"tag":46,"props":882,"children":883},{"style":731},[884],{"type":20,"value":734},{"type":14,"tag":46,"props":886,"children":887},{"style":737},[888],{"type":20,"value":889},"email\n",{"type":14,"tag":46,"props":891,"children":892},{"class":48,"line":148},[893,897,902,906],{"type":14,"tag":46,"props":894,"children":895},{"style":731},[896],{"type":20,"value":875},{"type":14,"tag":46,"props":898,"children":899},{"style":725},[900],{"type":20,"value":901},"unique",{"type":14,"tag":46,"props":903,"children":904},{"style":731},[905],{"type":20,"value":734},{"type":14,"tag":46,"props":907,"children":908},{"style":737},[909],{"type":20,"value":910},"customer_id\n",{"type":14,"tag":46,"props":912,"children":913},{"class":48,"line":323},[914],{"type":14,"tag":46,"props":915,"children":916},{"style":731},[917],{"type":20,"value":109},{"type":14,"tag":46,"props":919,"children":920},{"class":48,"line":332},[921,925,929,933],{"type":14,"tag":46,"props":922,"children":923},{"style":731},[924],{"type":20,"value":798},{"type":14,"tag":46,"props":926,"children":927},{"style":725},[928],{"type":20,"value":728},{"type":14,"tag":46,"props":930,"children":931},{"style":731},[932],{"type":20,"value":734},{"type":14,"tag":46,"props":934,"children":935},{"style":737},[936],{"type":20,"value":937},"predict\n",{"type":14,"tag":46,"props":939,"children":940},{"class":48,"line":341},[941,946,950],{"type":14,"tag":46,"props":942,"children":943},{"style":725},[944],{"type":20,"value":945},"    model",{"type":14,"tag":46,"props":947,"children":948},{"style":731},[949],{"type":20,"value":734},{"type":14,"tag":46,"props":951,"children":952},{"style":737},[953],{"type":20,"value":954},"churn_v1.0\n",{"type":14,"tag":46,"props":956,"children":957},{"class":48,"line":349},[958,963,967],{"type":14,"tag":46,"props":959,"children":960},{"style":725},[961],{"type":20,"value":962},"    input",{"type":14,"tag":46,"props":964,"children":965},{"style":731},[966],{"type":20,"value":734},{"type":14,"tag":46,"props":968,"children":969},{"style":737},[970],{"type":20,"value":971},"extract.output\n",{"type":14,"tag":46,"props":973,"children":974},{"class":48,"line":358},[975],{"type":14,"tag":46,"props":976,"children":977},{"style":731},[978],{"type":20,"value":109},{"type":14,"tag":46,"props":980,"children":981},{"class":48,"line":367},[982,986,990,994],{"type":14,"tag":46,"props":983,"children":984},{"style":731},[985],{"type":20,"value":798},{"type":14,"tag":46,"props":987,"children":988},{"style":725},[989],{"type":20,"value":728},{"type":14,"tag":46,"props":991,"children":992},{"style":731},[993],{"type":20,"value":734},{"type":14,"tag":46,"props":995,"children":996},{"style":737},[997],{"type":20,"value":998},"load\n",{"type":14,"tag":46,"props":1000,"children":1001},{"class":48,"line":376},[1002,1007,1011],{"type":14,"tag":46,"props":1003,"children":1004},{"style":725},[1005],{"type":20,"value":1006},"    destination",{"type":14,"tag":46,"props":1008,"children":1009},{"style":731},[1010],{"type":20,"value":734},{"type":14,"tag":46,"props":1012,"children":1013},{"style":737},[1014],{"type":20,"value":1015},"predictions_daily\n",{"type":14,"tag":46,"props":1017,"children":1018},{"class":48,"line":607},[1019,1023,1027],{"type":14,"tag":46,"props":1020,"children":1021},{"style":725},[1022],{"type":20,"value":962},{"type":14,"tag":46,"props":1024,"children":1025},{"style":731},[1026],{"type":20,"value":734},{"type":14,"tag":46,"props":1028,"children":1029},{"style":737},[1030],{"type":20,"value":1031},"predict.output\n",{"type":14,"tag":46,"props":1033,"children":1035},{"class":48,"line":1034},21,[1036],{"type":14,"tag":46,"props":1037,"children":1038},{"style":731},[1039],{"type":20,"value":109},{"type":14,"tag":46,"props":1041,"children":1043},{"class":48,"line":1042},22,[1044,1048,1052,1056],{"type":14,"tag":46,"props":1045,"children":1046},{"style":731},[1047],{"type":20,"value":798},{"type":14,"tag":46,"props":1049,"children":1050},{"style":725},[1051],{"type":20,"value":728},{"type":14,"tag":46,"props":1053,"children":1054},{"style":731},[1055],{"type":20,"value":734},{"type":14,"tag":46,"props":1057,"children":1058},{"style":737},[1059],{"type":20,"value":1060},"report\n",{"type":14,"tag":46,"props":1062,"children":1064},{"class":48,"line":1063},23,[1065,1070,1074],{"type":14,"tag":46,"props":1066,"children":1067},{"style":725},[1068],{"type":20,"value":1069},"    template",{"type":14,"tag":46,"props":1071,"children":1072},{"style":731},[1073],{"type":20,"value":734},{"type":14,"tag":46,"props":1075,"children":1076},{"style":737},[1077],{"type":20,"value":1078},"churn_daily_report\n",{"type":14,"tag":46,"props":1080,"children":1082},{"class":48,"line":1081},24,[1083,1088,1093,1098],{"type":14,"tag":46,"props":1084,"children":1085},{"style":725},[1086],{"type":20,"value":1087},"    recipients",{"type":14,"tag":46,"props":1089,"children":1090},{"style":731},[1091],{"type":20,"value":1092},": [",{"type":14,"tag":46,"props":1094,"children":1095},{"style":737},[1096],{"type":20,"value":1097},"team@company.com",{"type":14,"tag":46,"props":1099,"children":1100},{"style":731},[1101],{"type":20,"value":1102},"]\n",{"type":14,"tag":22,"props":1104,"children":1105},{},[1106,1110],{"type":14,"tag":165,"props":1107,"children":1108},{},[1109],{"type":20,"value":390},{"type":20,"value":1111}," Pipeline running → Track data flow with Lineage",{"type":14,"tag":28,"props":1113,"children":1115},{"id":1114},"phase-4-track-with-lineage",[1116],{"type":20,"value":1117},"Phase 4: Track with Lineage",{"type":14,"tag":22,"props":1119,"children":1120},{},[1121],{"type":14,"tag":165,"props":1122,"children":1123},{},[1124],{"type":20,"value":1125},"Use alongside Orchestration to:",{"type":14,"tag":171,"props":1127,"children":1128},{},[1129,1134,1139,1144],{"type":14,"tag":175,"props":1130,"children":1131},{},[1132],{"type":20,"value":1133},"Understand data flow",{"type":14,"tag":175,"props":1135,"children":1136},{},[1137],{"type":20,"value":1138},"Debug data issues",{"type":14,"tag":175,"props":1140,"children":1141},{},[1142],{"type":20,"value":1143},"See impact of changes",{"type":14,"tag":175,"props":1145,"children":1146},{},[1147],{"type":20,"value":1148},"Ensure quality",{"type":14,"tag":196,"props":1150,"children":1152},{"id":1151},"lineage-maps-your-data-journey",[1153],{"type":20,"value":1154},"Lineage Maps Your Data Journey",{"type":14,"tag":35,"props":1156,"children":1158},{"code":1157},"Raw Customer DB\n    ↓ (Extract)\nCleaned Data\n    ↓ (Transform)\nFeatures Table\n    ↓ (ML Model)\nPredictions\n    ↓ (Load)\nProduction Ready\n    ↓ (Reports & Dashboards)\nBusiness Insights\n",[1159],{"type":14,"tag":42,"props":1160,"children":1161},{"__ignoreMap":7},[1162],{"type":20,"value":1157},{"type":14,"tag":196,"props":1164,"children":1166},{"id":1165},"example-churn-prediction-lineage",[1167],{"type":20,"value":1168},"Example: Churn Prediction Lineage",{"type":14,"tag":35,"props":1170,"children":1172},{"code":1171},"Production Database\n├─ customers table\n│   └─ Extract step (daily)\n│       ↓\n├─ features_cleaned\n│   └─ Transform step (validation)\n│       ↓\n├─ model_churn_v1.0\n│   └─ Train (from ML Experiments)\n│       ↓\n├─ predictions\n│   └─ Score step (daily 9 AM)\n│       ↓\n└─ churn_risk_report\n    └─ Report generation\n        └─ Sent to Sales team\n",[1173],{"type":14,"tag":42,"props":1174,"children":1175},{"__ignoreMap":7},[1176],{"type":20,"value":1171},{"type":14,"tag":22,"props":1178,"children":1179},{},[1180],{"type":14,"tag":165,"props":1181,"children":1182},{},[1183],{"type":20,"value":1184},"Questions Lineage Answers:",{"type":14,"tag":35,"props":1186,"children":1188},{"code":1187},"\"Why are predictions different this week?\"\n→ Check lineage: Data source changed? Transform updated?\n\n\"Which reports use customer data?\"\n→ Check lineage: Trace forward from customers table\n\n\"If I delete this column, what breaks?\"\n→ Check lineage: See all downstream dependencies\n\n\"Where did this number come from?\"\n→ Check lineage: Trace backward to source\n",[1189],{"type":14,"tag":42,"props":1190,"children":1191},{"__ignoreMap":7},[1192],{"type":20,"value":1187},{"type":14,"tag":28,"props":1194,"children":1196},{"id":1195},"phase-5-document-in-metadata",[1197],{"type":20,"value":1198},"Phase 5: Document in Metadata",{"type":14,"tag":22,"props":1200,"children":1201},{},[1202],{"type":14,"tag":165,"props":1203,"children":1204},{},[1205],{"type":20,"value":1206},"Document everything for team reuse:",{"type":14,"tag":171,"props":1208,"children":1209},{},[1210,1215,1220,1225,1230],{"type":14,"tag":175,"props":1211,"children":1212},{},[1213],{"type":20,"value":1214},"What data exists",{"type":14,"tag":175,"props":1216,"children":1217},{},[1218],{"type":20,"value":1219},"How to use it",{"type":14,"tag":175,"props":1221,"children":1222},{},[1223],{"type":20,"value":1224},"Who owns it",{"type":14,"tag":175,"props":1226,"children":1227},{},[1228],{"type":20,"value":1229},"Data quality",{"type":14,"tag":175,"props":1231,"children":1232},{},[1233],{"type":20,"value":1234},"Business meaning",{"type":14,"tag":196,"props":1236,"children":1238},{"id":1237},"metadata-catalog",[1239],{"type":20,"value":1240},"Metadata Catalog",{"type":14,"tag":35,"props":1242,"children":1244},{"code":1243},"Asset: churn_predictions\nOwner: Data Science Team\nDescription: Daily churn risk scores for all customers\nUpdate frequency: Daily at 9:15 AM\n\nSchema:\n- customer_id: Unique customer identifier\n- churn_risk: Probability (0-100%)\n- risk_level: Category (low\u002Fmedium\u002Fhigh)\n\nQuality SLA: 99% completeness\nLast updated: 2024-06-12 09:15 UTC\n\nUsed by:\n- Sales Dashboard\n- Customer Retention Team\n- Marketing Campaigns\n\nRelated:\n- churn_model_v1.0 (ML model)\n- customers (source table)\n",[1245],{"type":14,"tag":42,"props":1246,"children":1247},{"__ignoreMap":7},[1248],{"type":20,"value":1243},{"type":14,"tag":28,"props":1250,"children":1252},{"id":1251},"complete-example-end-to-end",[1253],{"type":20,"value":1254},"Complete Example: End-to-End",{"type":14,"tag":196,"props":1256,"children":1258},{"id":1257},"week-1-exploration-phase",[1259],{"type":20,"value":1260},"Week 1: Exploration Phase",{"type":14,"tag":35,"props":1262,"children":1264},{"code":1263},"Monday: Load customer data in Notebook\nTuesday: Analyze churn patterns\nWednesday: Test different models\nThursday: Simple visualization\nFriday: Present findings\n",[1265],{"type":14,"tag":42,"props":1266,"children":1267},{"__ignoreMap":7},[1268],{"type":20,"value":1263},{"type":14,"tag":196,"props":1270,"children":1272},{"id":1271},"week-2-experimentation-phase",[1273],{"type":20,"value":1274},"Week 2: Experimentation Phase",{"type":14,"tag":35,"props":1276,"children":1278},{"code":1277},"Monday: Set up ML Experiments\nTuesday-Thursday: Try 15 different model approaches\nFriday: Compare all runs, select best\n",[1279],{"type":14,"tag":42,"props":1280,"children":1281},{"__ignoreMap":7},[1282],{"type":20,"value":1277},{"type":14,"tag":196,"props":1284,"children":1286},{"id":1285},"week-3-production-phase",[1287],{"type":20,"value":1288},"Week 3: Production Phase",{"type":14,"tag":35,"props":1290,"children":1292},{"code":1291},"Monday: Build Orchestration pipeline\nTuesday: Test pipeline end-to-end\nWednesday: Run in production\nThursday: Verify results in database\nFriday: Set up dashboards from results\n",[1293],{"type":14,"tag":42,"props":1294,"children":1295},{"__ignoreMap":7},[1296],{"type":20,"value":1291},{"type":14,"tag":196,"props":1298,"children":1300},{"id":1299},"week-4-operations-phase",[1301],{"type":20,"value":1302},"Week 4: Operations Phase",{"type":14,"tag":35,"props":1304,"children":1306},{"code":1305},"Every day: Pipeline runs at 9 AM\nMonitor: Check Lineage for data quality\nUpdate: Metadata with new insights\nShare: Results with stakeholders\n",[1307],{"type":14,"tag":42,"props":1308,"children":1309},{"__ignoreMap":7},[1310],{"type":20,"value":1305},{"type":14,"tag":28,"props":1312,"children":1314},{"id":1313},"common-workflows",[1315],{"type":20,"value":1316},"Common Workflows",{"type":14,"tag":196,"props":1318,"children":1320},{"id":1319},"scenario-1-improving-an-existing-model",[1321],{"type":20,"value":1322},"Scenario 1: Improving an Existing Model",{"type":14,"tag":35,"props":1324,"children":1326},{"code":1325},"Current: ML model runs daily, accuracy 85%\n\nStep 1 (Notebook):\n- Load current predictions\n- Analyze errors\n- Identify patterns\n- Test improvements\n\nStep 2 (ML Experiments):\n- Try new feature engineering\n- Test new algorithm\n- Compare accuracy\n- New approach: 92%\n\nStep 3 (Orchestration):\n- Update pipeline with new model\n- Test end-to-end\n- Deploy when ready\n- Keep old as fallback\n\nStep 4 (Lineage):\n- Monitor data flows\n- Verify impact across dashboards\n- Document changes\n\nStep 5 (Metadata):\n- Update model documentation\n- Note improvement (85% → 92%)\n- Record change date\n",[1327],{"type":14,"tag":42,"props":1328,"children":1329},{"__ignoreMap":7},[1330],{"type":20,"value":1325},{"type":14,"tag":196,"props":1332,"children":1334},{"id":1333},"scenario-2-adding-new-data-source",[1335],{"type":20,"value":1336},"Scenario 2: Adding New Data Source",{"type":14,"tag":35,"props":1338,"children":1340},{"code":1339},"Business Need: Include recent transactions in churn model\n\nStep 1 (Notebook):\n- Load transactions data\n- Explore patterns\n- Test impact on model\n\nStep 2 (ML Experiments):\n- Retrain with new feature\n- Compare old vs new\n- Validate improvement\n\nStep 3 (Orchestration):\n- Add new query to pipeline\n- Extract transactions\n- Join with existing features\n\nStep 4 (Lineage):\n- See new data source\n- Verify it flows to predictions\n- Check downstream impact\n\nStep 5 (Metadata):\n- Document transactions table\n- Add to data catalog\n- Explain usage\n",[1341],{"type":14,"tag":42,"props":1342,"children":1343},{"__ignoreMap":7},[1344],{"type":20,"value":1339},{"type":14,"tag":196,"props":1346,"children":1348},{"id":1347},"scenario-3-debugging-data-issues",[1349],{"type":20,"value":1350},"Scenario 3: Debugging Data Issues",{"type":14,"tag":35,"props":1352,"children":1354},{"code":1353},"Problem: Dashboard shows wrong numbers\n\nStep 1 (Metadata):\n- Find data source\n- Check last update time\n- See data quality metrics\n\nStep 2 (Lineage):\n- Trace backward from dashboard\n- Find transformation that feeds it\n- Check for recent changes\n\nStep 3 (Orchestration):\n- Check pipeline logs\n- See if step failed\n- Look at error messages\n\nStep 4 (Notebook):\n- Load raw data\n- Verify manually\n- Test transformations\n- Find the issue\n\nStep 5 (Fix & Rerun):\n- Fix code\n- Rerun pipeline\n- Verify results\n- Update documentation\n",[1355],{"type":14,"tag":42,"props":1356,"children":1357},{"__ignoreMap":7},[1358],{"type":20,"value":1353},{"type":14,"tag":28,"props":1360,"children":1362},{"id":1361},"best-practices-across-phases",[1363],{"type":20,"value":1364},"Best Practices Across Phases",{"type":14,"tag":196,"props":1366,"children":1368},{"id":1367},"notebook-phase",[1369],{"type":20,"value":1370},"Notebook Phase",{"type":14,"tag":171,"props":1372,"children":1373},{},[1374,1379,1384,1389,1394],{"type":14,"tag":175,"props":1375,"children":1376},{},[1377],{"type":20,"value":1378},"✓ Use descriptive cell names",{"type":14,"tag":175,"props":1380,"children":1381},{},[1382],{"type":20,"value":1383},"✓ Document your thinking",{"type":14,"tag":175,"props":1385,"children":1386},{},[1387],{"type":20,"value":1388},"✓ Keep experiments organized",{"type":14,"tag":175,"props":1390,"children":1391},{},[1392],{"type":20,"value":1393},"✓ Save reusable code snippets",{"type":14,"tag":175,"props":1395,"children":1396},{},[1397],{"type":20,"value":1398},"✗ Don't push to production directly",{"type":14,"tag":196,"props":1400,"children":1402},{"id":1401},"experiments-phase",[1403],{"type":20,"value":1404},"Experiments Phase",{"type":14,"tag":171,"props":1406,"children":1407},{},[1408,1413,1418,1423,1428],{"type":14,"tag":175,"props":1409,"children":1410},{},[1411],{"type":20,"value":1412},"✓ Log all parameters",{"type":14,"tag":175,"props":1414,"children":1415},{},[1416],{"type":20,"value":1417},"✓ Compare fairly (same data)",{"type":14,"tag":175,"props":1419,"children":1420},{},[1421],{"type":20,"value":1422},"✓ Save model artifacts",{"type":14,"tag":175,"props":1424,"children":1425},{},[1426],{"type":20,"value":1427},"✓ Document assumptions",{"type":14,"tag":175,"props":1429,"children":1430},{},[1431],{"type":20,"value":1432},"✗ Don't use in production until tested",{"type":14,"tag":196,"props":1434,"children":1436},{"id":1435},"orchestration-phase",[1437],{"type":20,"value":1438},"Orchestration Phase",{"type":14,"tag":171,"props":1440,"children":1441},{},[1442,1447,1452,1457,1462],{"type":14,"tag":175,"props":1443,"children":1444},{},[1445],{"type":20,"value":1446},"✓ Test thoroughly first",{"type":14,"tag":175,"props":1448,"children":1449},{},[1450],{"type":20,"value":1451},"✓ Set up monitoring",{"type":14,"tag":175,"props":1453,"children":1454},{},[1455],{"type":20,"value":1456},"✓ Plan for failures",{"type":14,"tag":175,"props":1458,"children":1459},{},[1460],{"type":20,"value":1461},"✓ Document workflow",{"type":14,"tag":175,"props":1463,"children":1464},{},[1465],{"type":20,"value":1466},"✗ Don't change production pipelines without testing",{"type":14,"tag":196,"props":1468,"children":1470},{"id":1469},"lineage-metadata-phase",[1471],{"type":20,"value":1472},"Lineage & Metadata Phase",{"type":14,"tag":171,"props":1474,"children":1475},{},[1476,1481,1486,1491,1496],{"type":14,"tag":175,"props":1477,"children":1478},{},[1479],{"type":20,"value":1480},"✓ Keep documentation current",{"type":14,"tag":175,"props":1482,"children":1483},{},[1484],{"type":20,"value":1485},"✓ Document changes",{"type":14,"tag":175,"props":1487,"children":1488},{},[1489],{"type":20,"value":1490},"✓ Tag data appropriately",{"type":14,"tag":175,"props":1492,"children":1493},{},[1494],{"type":20,"value":1495},"✓ Set quality rules",{"type":14,"tag":175,"props":1497,"children":1498},{},[1499],{"type":20,"value":1500},"✗ Don't let documentation lag behind code",{"type":14,"tag":28,"props":1502,"children":1504},{"id":1503},"related-topics",[1505],{"type":20,"value":1506},"Related Topics",{"type":14,"tag":171,"props":1508,"children":1509},{},[1510,1522,1533,1544,1555],{"type":14,"tag":175,"props":1511,"children":1512},{},[1513,1520],{"type":14,"tag":1514,"props":1515,"children":1517},"a",{"href":1516},"\u002Fdocs\u002Ffeatures\u002Fnotebook",[1518],{"type":20,"value":1519},"Notebook",{"type":20,"value":1521}," - Start exploring",{"type":14,"tag":175,"props":1523,"children":1524},{},[1525,1531],{"type":14,"tag":1514,"props":1526,"children":1528},{"href":1527},"\u002Fdocs\u002Ffeatures\u002Fml-experiments",[1529],{"type":20,"value":1530},"ML Experiments",{"type":20,"value":1532}," - Compare approaches",{"type":14,"tag":175,"props":1534,"children":1535},{},[1536,1542],{"type":14,"tag":1514,"props":1537,"children":1539},{"href":1538},"\u002Fdocs\u002Ffeatures\u002Forchestration",[1540],{"type":20,"value":1541},"Orchestration",{"type":20,"value":1543}," - Automate workflows",{"type":14,"tag":175,"props":1545,"children":1546},{},[1547,1553],{"type":14,"tag":1514,"props":1548,"children":1550},{"href":1549},"\u002Fdocs\u002Ffeatures\u002Flineage",[1551],{"type":20,"value":1552},"Lineage",{"type":20,"value":1554}," - Track data flows",{"type":14,"tag":175,"props":1556,"children":1557},{},[1558,1564],{"type":14,"tag":1514,"props":1559,"children":1561},{"href":1560},"\u002Fdocs\u002Ffeatures\u002Fmetadata",[1562],{"type":20,"value":1563},"Metadata",{"type":20,"value":1565}," - Document data",{"type":14,"tag":1567,"props":1568,"children":1569},"style",{},[1570],{"type":20,"value":1571},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}",{"title":7,"searchDepth":58,"depth":58,"links":1573},[1574,1575,1579,1583,1587,1591,1594,1600,1605,1611],{"id":30,"depth":58,"text":33},{"id":157,"depth":58,"text":160,"children":1576},[1577,1578],{"id":198,"depth":67,"text":201},{"id":212,"depth":67,"text":215},{"id":395,"depth":58,"text":398,"children":1580},[1581,1582],{"id":432,"depth":67,"text":435},{"id":446,"depth":67,"text":449},{"id":641,"depth":58,"text":644,"children":1584},[1585,1586],{"id":677,"depth":67,"text":680},{"id":691,"depth":67,"text":694},{"id":1114,"depth":58,"text":1117,"children":1588},[1589,1590],{"id":1151,"depth":67,"text":1154},{"id":1165,"depth":67,"text":1168},{"id":1195,"depth":58,"text":1198,"children":1592},[1593],{"id":1237,"depth":67,"text":1240},{"id":1251,"depth":58,"text":1254,"children":1595},[1596,1597,1598,1599],{"id":1257,"depth":67,"text":1260},{"id":1271,"depth":67,"text":1274},{"id":1285,"depth":67,"text":1288},{"id":1299,"depth":67,"text":1302},{"id":1313,"depth":58,"text":1316,"children":1601},[1602,1603,1604],{"id":1319,"depth":67,"text":1322},{"id":1333,"depth":67,"text":1336},{"id":1347,"depth":67,"text":1350},{"id":1361,"depth":58,"text":1364,"children":1606},[1607,1608,1609,1610],{"id":1367,"depth":67,"text":1370},{"id":1401,"depth":67,"text":1404},{"id":1435,"depth":67,"text":1438},{"id":1469,"depth":67,"text":1472},{"id":1503,"depth":58,"text":1506},"markdown","content:docs:features:data-workflow.md","content","docs\u002Ffeatures\u002Fdata-workflow.md","docs\u002Ffeatures\u002Fdata-workflow","md",1782233761654]