{
  "page": {
    "id": "5sw7z2ldf26r",
    "name": "Commerce Layer",
    "url": "https://status.commercelayer.io"
  },
  "status": {
    "indicator": "none",
    "description": "All Systems Operational"
  },
  "components": [{"status":"operational","name":"Commerce API","created_at":"2018-05-03T07:43:20.605+02:00","updated_at":"2026-09-28T07:39:20.756+02:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"34km6s8trv84","group_id":null,"page_id":"5sw7z2ldf26r"},{"status":"operational","name":"Dashboard","created_at":"2018-05-03T07:45:17.394+02:00","updated_at":"2026-09-28T07:39:20.778+02:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"md7hnkpmydpq","group_id":null,"page_id":"5sw7z2ldf26r"},{"status":"operational","name":"Documentation","created_at":"2018-05-03T07:43:20.631+02:00","updated_at":"2026-03-24T12:00:07.299+01:00","position":3,"description":null,"showcase":true,"start_date":null,"id":"p2mg1311w47p","group_id":null,"page_id":"5sw7z2ldf26r"},{"status":"operational","name":"Metrics API","created_at":"2025-12-12T19:46:06.407+01:00","updated_at":"2025-12-12T19:46:27.201+01:00","position":4,"description":null,"showcase":true,"start_date":"1993-02-18T00:00:00.000+01:00","id":"xxh5b72s5mg8","group_id":null,"page_id":"5sw7z2ldf26r"},{"status":"operational","name":"Stream Hub","created_at":"2025-12-12T19:46:54.282+01:00","updated_at":"2026-09-28T07:39:20.798+02:00","position":5,"description":null,"showcase":true,"start_date":"1993-02-18T00:00:00.000+01:00","id":"t773tm86xm3d","group_id":null,"page_id":"5sw7z2ldf26r"},{"status":"operational","name":"Asynchronous Jobs","created_at":"2026-09-21T10:45:53.531+02:00","updated_at":"2026-09-28T07:39:20.817+02:00","position":6,"description":"e.g. Webhooks, Exports, Imports, etc.","showcase":true,"start_date":"2020-01-01T00:00:00.000+01:00","id":"tm5yn5d1hdgh","group_id":null,"page_id":"5sw7z2ldf26r"},{"status":"operational","name":"Apps","created_at":"2023-03-09T11:38:52.784+01:00","updated_at":"2026-09-21T10:46:08.289+02:00","position":7,"description":null,"showcase":false,"start_date":null,"id":"g5r1nct65t0v","group_id":null,"page_id":"5sw7z2ldf26r","components":["fr85x46mwqgg","hz5dt1kmlqzy","f1nqxqxcvcfj"]}],
  "incidents": [{"name":"Routine operation planned for 28/09/2026","status":"completed","created_at":"2026-09-21T11:16:34.725+02:00","updated_at":"2026-09-28T07:39:20.893+02:00","monitoring_at":null,"resolved_at":"2026-09-28T07:39:20.863+02:00","impact":"maintenance","shortlink":"https://stspg.io/m08sk0t1vs8g","scheduled_for":"2026-09-28T06:30:00.000+02:00","scheduled_until":"2026-09-28T07:30:00.000+02:00","scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":true,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-09-21T11:16:34.714+02:00","reminder_intervals":null,"id":"23wkr992z535","page_id":"5sw7z2ldf26r","incident_updates":[{"status":"completed","body":"System updates have been applied. Our services are fully operational.","created_at":"2026-09-28T07:39:20.863+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-09-28T07:39:20.863+02:00","display_at":"2026-09-28T07:39:20.863+02:00","deliver_notifications":true,"tweet_id":null,"id":"s5z2kfxk7q4h","incident_id":"23wkr992z535","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"under_maintenance","new_status":"operational"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"under_maintenance","new_status":"operational"},{"code":"t773tm86xm3d","name":"Stream Hub","old_status":"under_maintenance","new_status":"operational"},{"code":"tm5yn5d1hdgh","name":"Asynchronous Jobs","old_status":"under_maintenance","new_status":"operational"}]},{"status":"in_progress","body":"Scheduled maintenance is currently in progress. We will provide updates as necessary.","created_at":"2026-09-28T06:30:23.489+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-09-28T06:30:23.489+02:00","display_at":"2026-09-28T06:30:23.489+02:00","deliver_notifications":true,"tweet_id":null,"id":"xtwx6p0nnh6h","incident_id":"23wkr992z535","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"operational","new_status":"under_maintenance"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"under_maintenance"},{"code":"t773tm86xm3d","name":"Stream Hub","old_status":"operational","new_status":"under_maintenance"},{"code":"tm5yn5d1hdgh","name":"Asynchronous Jobs","old_status":"operational","new_status":"under_maintenance"}]},{"status":"scheduled","body":"A routine operation is scheduled from 06:30 am to 07:30 am (CEST) on 28/09/2026. \nWe're applying the latest system updates to keep our infrastructure secure and up to date.\nPlease note that a small number of errors may occur for less than a minute. We’ll monitor the process and provide an update once the activity is complete.","created_at":"2026-09-21T11:16:34.813+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-09-21T11:16:34.813+02:00","display_at":"2026-09-21T11:16:34.813+02:00","deliver_notifications":true,"tweet_id":null,"id":"frskb3wf4g6d","incident_id":"23wkr992z535","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"operational","new_status":"operational"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"operational"},{"code":"t773tm86xm3d","name":"Stream Hub","old_status":"operational","new_status":"operational"},{"code":"tm5yn5d1hdgh","name":"Asynchronous Jobs","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Commerce API","created_at":"2018-05-03T07:43:20.605+02:00","updated_at":"2026-09-28T07:39:20.756+02:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"34km6s8trv84","page_id":"5sw7z2ldf26r","group_id":null},{"status":"operational","name":"Dashboard","created_at":"2018-05-03T07:45:17.394+02:00","updated_at":"2026-09-28T07:39:20.778+02:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"md7hnkpmydpq","page_id":"5sw7z2ldf26r","group_id":null},{"status":"operational","name":"Stream Hub","created_at":"2025-12-12T19:46:54.282+01:00","updated_at":"2026-09-28T07:39:20.798+02:00","position":5,"description":null,"showcase":true,"start_date":"1993-02-18T00:00:00.000+01:00","id":"t773tm86xm3d","page_id":"5sw7z2ldf26r","group_id":null},{"status":"operational","name":"Asynchronous Jobs","created_at":"2026-09-21T10:45:53.531+02:00","updated_at":"2026-09-28T07:39:20.817+02:00","position":6,"description":"e.g. Webhooks, Exports, Imports, etc.","showcase":true,"start_date":"2020-01-01T00:00:00.000+01:00","id":"tm5yn5d1hdgh","page_id":"5sw7z2ldf26r","group_id":null}]},{"name":"Incident report","status":"resolved","created_at":"2026-08-03T07:12:14.000+02:00","updated_at":"2026-08-04T10:47:24.925+02:00","monitoring_at":null,"resolved_at":"2026-08-03T07:12:14.000+02:00","impact":"critical","shortlink":"https://stspg.io/b62ggkj9hq3d","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"critical","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-08-03T07:12:14.000+02:00","reminder_intervals":"[]","id":"zk8cmyf0tbpk","page_id":"5sw7z2ldf26r","incident_updates":[{"status":"resolved","body":"During a routine automated update of the EU database, the procedure couldn't complete and the DB couldn't return to normal operations.\nOur monitoring systems alerted immediately our team, that could restart the procedure manually and completed it in a few minutes.\nRoot cause analysis currently in progress.","created_at":"2026-08-03T09:29:14.966+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-08-03T09:29:29.323+02:00","display_at":"2026-08-03T07:12:14.000+02:00","deliver_notifications":false,"tweet_id":null,"id":"63vz9pvgdtcm","incident_id":"zk8cmyf0tbpk","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"operational","new_status":"operational"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"operational"}]}],"postmortem_body":"# Summary\n\nOn Aug 3rd, 2026, from \\`5:02 AM UTC\\` till \\`5:13 AM UTC\\`, a routine automated update process failed to complete, causing a major downtime of our EU Database cluster.  \nMonitoring system alerted immediately our team, that started a manual override of the process, taking it to a successful end.\n\n# Leadup\n\nWhile our recurrent automated DB update process was running on the EU database it encountered an unexpected condition, that prevented the process to complete, leaving the cluster in an unconsistent state.\n\n# Fault\n\nSome database engine items were not in the expected condition for the update to complete correctly and that caused the update to fail.\n\n# Detection\n\nMoments after the initial failure of the automated process, our monitoring system alerted our infrastructure team.\n\n# Root Cause\n\nLogical replication feature was not in the proper state after some previous replication activity in the cluster.  \nThe current update process couldn't detect this situation in advance to avoid starting at all.\n\n# Mitigation and resolution\n\nA few minutes after the beginning of the issue, our operators were able to reset the inconsistent parameters and apply the manual update procedure.\n\n# Corrective and Preventative Measures\n\nWe are introducing a more robust update process that, by design, will not be prone to similar issues and will consistently limit the overall risk of errors during to database updates.","postmortem_body_last_updated_at":"2026-08-04T10:47:24.910+02:00","postmortem_ignored":false,"postmortem_published_at":"2026-08-04T10:47:24.910+02:00","postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Commerce API","created_at":"2018-05-03T07:43:20.605+02:00","updated_at":"2026-09-28T07:39:20.756+02:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"34km6s8trv84","page_id":"5sw7z2ldf26r","group_id":null},{"status":"operational","name":"Dashboard","created_at":"2018-05-03T07:45:17.394+02:00","updated_at":"2026-09-28T07:39:20.778+02:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"md7hnkpmydpq","page_id":"5sw7z2ldf26r","group_id":null}]},{"name":"Documentation upgrade","status":"completed","created_at":"2026-03-17T11:31:51.702+01:00","updated_at":"2026-03-24T12:00:07.394+01:00","monitoring_at":null,"resolved_at":"2026-03-24T12:00:07.359+01:00","impact":"maintenance","shortlink":"https://stspg.io/r436rsy4zq8n","scheduled_for":"2026-03-24T11:00:00.000+01:00","scheduled_until":"2026-03-24T12:00:00.000+01:00","scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":true,"scheduled_auto_completed":true,"metadata":{},"started_at":"2026-03-17T11:31:51.696+01:00","reminder_intervals":null,"id":"dc8z56409ywg","page_id":"5sw7z2ldf26r","incident_updates":[{"status":"completed","body":"The scheduled maintenance has been completed.","created_at":"2026-03-24T12:00:07.359+01:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-03-24T12:00:07.359+01:00","display_at":"2026-03-24T12:00:07.359+01:00","deliver_notifications":false,"tweet_id":null,"id":"5y3ch22k3nc1","incident_id":"dc8z56409ywg","custom_tweet":null,"affected_components":[{"code":"p2mg1311w47p","name":"Documentation","old_status":"under_maintenance","new_status":"operational"}]},{"status":"in_progress","body":"Scheduled maintenance is currently in progress. We will provide updates as necessary.","created_at":"2026-03-24T11:00:07.155+01:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-03-24T11:00:07.155+01:00","display_at":"2026-03-24T11:00:07.155+01:00","deliver_notifications":false,"tweet_id":null,"id":"pyl1r5rnj694","incident_id":"dc8z56409ywg","custom_tweet":null,"affected_components":[{"code":"p2mg1311w47p","name":"Documentation","old_status":"operational","new_status":"under_maintenance"}]},{"status":"scheduled","body":"We're upgrading our documentation. On March 24, some content may be unavailable from 11:00 AM to 12:00 AM CET while we complete the migration.","created_at":"2026-03-17T11:31:51.771+01:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-03-17T11:31:51.771+01:00","display_at":"2026-03-17T11:31:51.771+01:00","deliver_notifications":true,"tweet_id":null,"id":"js2x1s79x19g","incident_id":"dc8z56409ywg","custom_tweet":null,"affected_components":[{"code":"p2mg1311w47p","name":"Documentation","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Documentation","created_at":"2018-05-03T07:43:20.631+02:00","updated_at":"2026-03-24T12:00:07.299+01:00","position":3,"description":null,"showcase":true,"start_date":null,"id":"p2mg1311w47p","page_id":"5sw7z2ldf26r","group_id":null}]},{"name":"Dashboard suffered from third party CDN issue","status":"resolved","created_at":"2025-11-18T13:20:35.000+01:00","updated_at":"2025-11-19T09:34:36.392+01:00","monitoring_at":"2025-11-18T13:20:35.000+01:00","resolved_at":"2025-11-19T09:34:36.376+01:00","impact":"minor","shortlink":"https://stspg.io/186l4k544gp7","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-11-18T13:20:35.000+01:00","reminder_intervals":null,"id":"wpvfydkz7j0q","page_id":"5sw7z2ldf26r","incident_updates":[{"status":"resolved","body":"The issue has been resolved.","created_at":"2025-11-19T09:34:36.376+01:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-11-19T09:34:36.376+01:00","display_at":"2025-11-19T09:34:36.376+01:00","deliver_notifications":false,"tweet_id":null,"id":"l9mc3bhcskfw","incident_id":"wpvfydkz7j0q","custom_tweet":null,"affected_components":[{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"degraded_performance","new_status":"operational"}]},{"status":"monitoring","body":"The CDN provider has confirmed a fix has been implemented. The issue seems to be resolved, but we'll continue to monitor the situation before closing the incident.","created_at":"2025-11-18T15:56:38.936+01:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-11-18T15:56:38.936+01:00","display_at":"2025-11-18T15:56:38.936+01:00","deliver_notifications":true,"tweet_id":null,"id":"rwr77q01slml","incident_id":"wpvfydkz7j0q","custom_tweet":null,"affected_components":[{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"degraded_performance","new_status":"degraded_performance"}]},{"status":"monitoring","body":"The Dashboard was not accessible for a few minutes. This is due to a currently ongoing incident on a third party CDN (https://www.cloudflarestatus.com/incidents/8gmgl950y3h7).  They are actively investigating and working on a fix. The dashboard looks accessible in this moment, but we cannot exclude other disservices until the issue is not resolved on their side. We'll keep the situation monitored.","created_at":"2025-11-18T13:20:35.702+01:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-11-18T15:57:04.837+01:00","display_at":"2025-11-18T13:20:35.000+01:00","deliver_notifications":true,"tweet_id":null,"id":"589vdgn78z8j","incident_id":"wpvfydkz7j0q","custom_tweet":null,"affected_components":[{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"degraded_performance"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Dashboard","created_at":"2018-05-03T07:45:17.394+02:00","updated_at":"2026-09-28T07:39:20.778+02:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"md7hnkpmydpq","page_id":"5sw7z2ldf26r","group_id":null}]},{"name":"Incident Report","status":"resolved","created_at":"2025-05-23T08:00:00.000+02:00","updated_at":"2025-06-11T16:39:29.061+02:00","monitoring_at":null,"resolved_at":"2025-05-23T08:00:00.000+02:00","impact":"major","shortlink":"https://stspg.io/mjw3zjvkhss4","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-05-23T08:00:00.000+02:00","reminder_intervals":null,"id":"k9y4x3h885d6","page_id":"5sw7z2ldf26r","incident_updates":[{"status":"resolved","body":"After a scheduled deployment, some customers experienced mismatched data responses on their interactions with our services.\nAfter detecting a major issue, we initiated the roll-back procedure, reverting our services to the previous code version.\nRoot cause analysis currently in progress.","created_at":"2025-05-26T19:11:42.177+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-05-27T11:59:22.834+02:00","display_at":"2025-05-23T08:00:00.000+02:00","deliver_notifications":false,"tweet_id":null,"id":"cf5kgcfr8t6p","incident_id":"k9y4x3h885d6","custom_tweet":null,"affected_components":null}],"postmortem_body":"# **Summary**\n\nOn May 23rd, 2025, from \\`5:50 AM UTC\\` till \\`7:03 AM UTC\\`, after a scheduled deployment,  some customers experienced mismatched data responses on their interactions with our services.\n\nRealizing that it was a major issue, we initiated the roll-back procedure, reverting our services to the previous code version.\n\n# **Leadup**\n\nAt  \\`5:50 AM UTC\\` we concluded a deployment that included an upgrade of a third party provider library. The deployment introduced errors on several sessions.\n\n# **Fault**\n\nSome sessions with tokens requested during the affected timeframe might have experienced issues with add to cart functionality and checkout. In particular, adding items to cart and placing orders was not always feasible. In addition to this, some placed orders were missing approved\\_at and reference data.\n\n# **Detection**\n\nA few minutes after the release, one of our operators detected an anomaly on the internal metrics dashboard that was confirmed by our clients requests to the support team.\n\n# **Root Cause**\n\nThe upgrade of a third party provider library that manages low level web request wrappers introduced problems in our APIs that have not been detected by our tests in preprod and qa environments. The upgrade was recommended to keep our services up to speed with the latest security patches and compatibility with other components.\n\n# **Mitigation and resolution**\n\nA few minutes after the release, once we realized the severity of the issue, we decided to roll-back to the previous version of the code in order to immediately restore normal operations and allow a deeper understanding of the root cause offline. In addition to the roll-back, a partial CDN cache purge was performed to reduce the TTL of the wrong data.\n\n# **Corrective and Preventative Measures**\n\nWe are introducing additional automatic tests related to the involved component and other similar patterns.","postmortem_body_last_updated_at":"2025-06-05T10:24:20.533+02:00","postmortem_ignored":false,"postmortem_published_at":"2025-05-27T10:18:04.203+02:00","postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]},{"name":"Fix has been applied - monitoring","status":"resolved","created_at":"2024-10-01T10:37:25.120+02:00","updated_at":"2024-10-01T10:44:09.214+02:00","monitoring_at":"2024-10-01T10:37:25.167+02:00","resolved_at":"2024-10-01T10:44:09.202+02:00","impact":"none","shortlink":"https://stspg.io/j2k8lz3dnkqx","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-10-01T10:37:25.114+02:00","reminder_intervals":null,"id":"fqjgdmmmlk8r","page_id":"5sw7z2ldf26r","incident_updates":[{"status":"resolved","body":"The Dashboard is now completely operational, the test/live toggle that disappeared, is now visible and clickable.","created_at":"2024-10-01T10:44:09.202+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-10-01T10:44:09.202+02:00","display_at":"2024-10-01T10:44:09.202+02:00","deliver_notifications":true,"tweet_id":null,"id":"whzd8xrgjn14","incident_id":"fqjgdmmmlk8r","custom_tweet":null,"affected_components":[{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"operational"}]},{"status":"monitoring","body":"The Dashboard was not completely accessible for few minutes. A fix has been applied and we are monitoring the situation. Api was never affected and have continued to respond as usual.","created_at":"2024-10-01T10:37:25.167+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-10-01T10:37:25.167+02:00","display_at":"2024-10-01T10:37:25.167+02:00","deliver_notifications":true,"tweet_id":null,"id":"tdm09nh1qlc0","incident_id":"fqjgdmmmlk8r","custom_tweet":null,"affected_components":[{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Dashboard","created_at":"2018-05-03T07:45:17.394+02:00","updated_at":"2026-09-28T07:39:20.778+02:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"md7hnkpmydpq","page_id":"5sw7z2ldf26r","group_id":null}]},{"name":"Scheduled maintenance planned for June 15, 2024","status":"completed","created_at":"2024-05-28T18:00:25.197+02:00","updated_at":"2024-06-15T05:26:42.175+02:00","monitoring_at":null,"resolved_at":"2024-06-15T05:26:42.161+02:00","impact":"maintenance","shortlink":"https://stspg.io/df42ckc5fydx","scheduled_for":"2024-06-15T05:00:00.000+02:00","scheduled_until":"2024-06-15T06:00:00.000+02:00","scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-05-28T18:00:25.187+02:00","reminder_intervals":null,"id":"d1ms30w1sp7h","page_id":"5sw7z2ldf26r","incident_updates":[{"status":"completed","body":"The upgrade is now successfully completed. Our APIs are fully operational.","created_at":"2024-06-15T05:26:42.161+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-06-15T05:26:42.161+02:00","display_at":"2024-06-15T05:26:42.161+02:00","deliver_notifications":false,"tweet_id":null,"id":"ygfy7jpr2c42","incident_id":"d1ms30w1sp7h","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"under_maintenance","new_status":"operational"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"under_maintenance","new_status":"operational"},{"code":"fr85x46mwqgg","name":"Apps - Checkout","old_status":"under_maintenance","new_status":"operational"}]},{"status":"in_progress","body":"Please note that our APIs could be partially unavailable for a few seconds in the upcoming minutes. This maintenance is to ensure that our infrastructure is updated with the latest security measures. During this time, you may get errors while trying to call our APIs. Async requests will not be affected.","created_at":"2024-06-15T04:59:10.167+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-06-15T04:59:10.167+02:00","display_at":"2024-06-15T04:59:10.167+02:00","deliver_notifications":false,"tweet_id":null,"id":"qdj2399g4z61","incident_id":"d1ms30w1sp7h","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"operational","new_status":"under_maintenance"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"under_maintenance"},{"code":"fr85x46mwqgg","name":"Apps - Checkout","old_status":"operational","new_status":"under_maintenance"}]},{"status":"scheduled","body":"Please note that our APIs could be partially unavailable for a few minutes, from 05:00 am to 06:00 am CET time on Saturday June 15, 2024. This maintenance is to ensure that our infrastructure is updated with the latest security measures. During this time, you may get errors while trying to call our APIs. Async requests will not be affected.","created_at":"2024-05-28T18:00:25.249+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-05-28T18:00:25.249+02:00","display_at":"2024-05-28T18:00:25.249+02:00","deliver_notifications":true,"tweet_id":null,"id":"9qclpwc6f4kk","incident_id":"d1ms30w1sp7h","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"operational","new_status":"operational"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"operational"},{"code":"fr85x46mwqgg","name":"Apps - Checkout","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Commerce API","created_at":"2018-05-03T07:43:20.605+02:00","updated_at":"2026-09-28T07:39:20.756+02:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"34km6s8trv84","page_id":"5sw7z2ldf26r","group_id":null},{"status":"operational","name":"Checkout","created_at":"2023-03-09T11:38:52.815+01:00","updated_at":"2024-06-15T05:26:42.126+02:00","position":1,"description":null,"showcase":true,"start_date":"2022-10-01T00:00:00.000+02:00","id":"fr85x46mwqgg","page_id":"5sw7z2ldf26r","group_id":"g5r1nct65t0v"},{"status":"operational","name":"Dashboard","created_at":"2018-05-03T07:45:17.394+02:00","updated_at":"2026-09-28T07:39:20.778+02:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"md7hnkpmydpq","page_id":"5sw7z2ldf26r","group_id":null}]},{"name":"API/Dashboard/Applications back to normal","status":"resolved","created_at":"2024-04-09T09:25:56.468+02:00","updated_at":"2024-04-22T16:09:25.753+02:00","monitoring_at":"2024-04-09T11:06:01.503+02:00","resolved_at":"2024-04-09T13:14:19.083+02:00","impact":"critical","shortlink":"https://stspg.io/n5rq2x9kx1dt","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-04-09T09:25:56.459+02:00","reminder_intervals":null,"id":"zr2t0kfxdbsg","page_id":"5sw7z2ldf26r","incident_updates":[{"status":"resolved","body":"The issue is resolved and the incident, closed. We're working on the root cause analysis, a post-mortem report will be available soon.","created_at":"2024-04-09T13:14:19.083+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-09T13:14:19.083+02:00","display_at":"2024-04-09T13:14:19.083+02:00","deliver_notifications":false,"tweet_id":null,"id":"h14p5xgw6qym","incident_id":"zr2t0kfxdbsg","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"operational","new_status":"operational"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"operational"},{"code":"fr85x46mwqgg","name":"Apps - Checkout","old_status":"operational","new_status":"operational"}]},{"status":"monitoring","body":"A fix has been applied.  We are monitoring the situation and will publish a detailed incident report.","created_at":"2024-04-09T11:06:01.503+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-09T11:06:01.503+02:00","display_at":"2024-04-09T11:06:01.503+02:00","deliver_notifications":false,"tweet_id":null,"id":"7mksxczrkwyt","incident_id":"zr2t0kfxdbsg","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"major_outage","new_status":"operational"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"major_outage","new_status":"operational"},{"code":"fr85x46mwqgg","name":"Apps - Checkout","old_status":"major_outage","new_status":"operational"}]},{"status":"identified","body":"We are working with our service provider to resolve the issue and restore the normal operations.","created_at":"2024-04-09T10:36:17.520+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-09T10:36:17.520+02:00","display_at":"2024-04-09T10:36:17.520+02:00","deliver_notifications":false,"tweet_id":null,"id":"8gksps2rwlrm","incident_id":"zr2t0kfxdbsg","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"major_outage","new_status":"major_outage"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"major_outage","new_status":"major_outage"},{"code":"fr85x46mwqgg","name":"Apps - Checkout","old_status":"major_outage","new_status":"major_outage"}]},{"status":"investigating","body":"We are continuing to investigate this issue.","created_at":"2024-04-09T09:26:20.117+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-09T09:26:20.117+02:00","display_at":"2024-04-09T09:26:20.117+02:00","deliver_notifications":false,"tweet_id":null,"id":"x80zs9bw1yf7","incident_id":"zr2t0kfxdbsg","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"operational","new_status":"major_outage"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"major_outage"},{"code":"fr85x46mwqgg","name":"Apps - Checkout","old_status":"operational","new_status":"major_outage"}]},{"status":"investigating","body":"We are aware that our API/Dashboard/Applications are currently not available. We are investigating it with the highest priority and will provide updates here.","created_at":"2024-04-09T09:25:56.542+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-09T09:25:56.542+02:00","display_at":"2024-04-09T09:25:56.542+02:00","deliver_notifications":false,"tweet_id":null,"id":"5g0wchdpfjwt","incident_id":"zr2t0kfxdbsg","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"operational","new_status":"operational"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"operational"},{"code":"fr85x46mwqgg","name":"Apps - Checkout","old_status":"operational","new_status":"operational"}]}],"postmortem_body":"# **Summary**\n\nOn April 9th, from 6:45 AM to 8:55 AM UTC, during the scheduled maintenance window, we encountered a database issue. The database version upgrade process failed and the automatic recovery service provided by our cloud infrastructure provider did not activate. We immediately contacted the infrastructure provider’s support team. They addressed this problem and there was no resulting data loss.\n\n# **Leadup**\n\nOn April 9th, 2024, we began the DB upgrade maintenance at 6:39 AM UTC. At approximately 6:45 AM UTC, the background task provided by the DB cloud service failed, leaving the entire upgrade procedure stuck in an intermediate state.\n\n# **Fault**\n\nDuring the incident, all Core API calls resulted in errors. These errors were spread across all endpoints and customers, depending on their traffic distribution and type. The read requests cached on our CDN continued to work without errors.\n\n# **Detection**\n\nA few minutes after beginning maintenance, our engineers realized that the automatic upgrade process was not responding. The cloud service provides safeguards during upgrades that automatically restores normal operations should a timeout exceed 5 minutes. This expected action did not occur.    \n\n# **Root Cause**\n\nThe service provider's support team confirmed that the upgrade process, once initiated, encountered an unexpected issue within their internal procedure, causing the upgrade to stall.\n\nThe automatic recovery safeguard began as expected five minutes after the incident. However, it couldn't complete because one of the read replicas was in an inconsistent state. The vendor is still investigating the cause of this issue.\n\nSince the database service is fully managed by the provider, our engineers had to wait for the provider's support team to restore the normal level of availability.\n\n# **Mitigation and resolution**\n\nA few minutes before 7:00 AM UTC, our engineers tried to manually interrupt the process, but the option to cancel it was disabled by the vendor. Our team immediately engaged the service provider’s support team, requesting to interrupt the process and restore the availability of the DB, with the highest urgency.\n\nThe provider’s support team started investigating through their own tools and after a few minutes suggested a couple of workarounds that didn’t succeed.\n\nIn parallel, we began our own restoration process from a backup.  \n\nAt about 8:50 AM UTC, the service provider’s support team was able to restore the DB availability and our services gradually recovered to normal operation levels. At approximately 8:55 AM UTC, the entire resolution was completed.\n\n# **Corrective and Preventative Measures**\n\nGiven the nature of the cloud infrastructure on which our services are built upon, not all operational steps are in our full control. However, we identified improvements that we intend to implement on processes and procedures, together with our service provider: \n\n* We will directly involve the service provider’s representatives in future maintenance operations, from planning through implementation.\n* We will update our status page as soon as the issue occurs. We will extend this by sending a proactive alert to organization owners.  \n* We will schedule the maintenance window during a low traffic period on our platform.","postmortem_body_last_updated_at":"2024-04-22T16:09:25.737+02:00","postmortem_ignored":false,"postmortem_published_at":"2024-04-22T16:09:25.737+02:00","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Commerce API","created_at":"2018-05-03T07:43:20.605+02:00","updated_at":"2026-09-28T07:39:20.756+02:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"34km6s8trv84","page_id":"5sw7z2ldf26r","group_id":null},{"status":"operational","name":"Checkout","created_at":"2023-03-09T11:38:52.815+01:00","updated_at":"2024-06-15T05:26:42.126+02:00","position":1,"description":null,"showcase":true,"start_date":"2022-10-01T00:00:00.000+02:00","id":"fr85x46mwqgg","page_id":"5sw7z2ldf26r","group_id":"g5r1nct65t0v"},{"status":"operational","name":"Dashboard","created_at":"2018-05-03T07:45:17.394+02:00","updated_at":"2026-09-28T07:39:20.778+02:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"md7hnkpmydpq","page_id":"5sw7z2ldf26r","group_id":null}]},{"name":"Scheduled maintenance planned for April 8 and 9, 2024","status":"completed","created_at":"2024-04-08T10:58:16.861+02:00","updated_at":"2024-04-09T12:08:20.412+02:00","monitoring_at":null,"resolved_at":"2024-04-09T12:08:20.394+02:00","impact":"maintenance","shortlink":"https://stspg.io/08f0btn7kpc7","scheduled_for":"2024-04-09T08:00:00.000+02:00","scheduled_until":"2024-04-09T10:00:00.000+02:00","scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-04-08T10:58:16.852+02:00","reminder_intervals":null,"id":"x3rdnh5mqb4x","page_id":"5sw7z2ldf26r","incident_updates":[{"status":"completed","body":"The upgrade is now successfully completed. Our APIs are fully operational.","created_at":"2024-04-09T12:08:20.394+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-09T12:08:20.394+02:00","display_at":"2024-04-09T12:08:20.394+02:00","deliver_notifications":false,"tweet_id":null,"id":"d5zj19jtn7t8","incident_id":"x3rdnh5mqb4x","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"operational","new_status":"operational"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"operational"},{"code":"fr85x46mwqgg","name":"Apps - Checkout","old_status":"operational","new_status":"operational"}]},{"status":"in_progress","body":"Please note that our APIs could be partially unavailable for two intervals of a few seconds in the upcoming couple of hours. This maintenance is to ensure that our infrastructure is updated with the latest security measures. During this time, you may get errors while trying to call our APIs. Async requests will not be affected.","created_at":"2024-04-09T08:11:37.335+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-09T08:11:37.335+02:00","display_at":"2024-04-09T08:11:37.335+02:00","deliver_notifications":false,"tweet_id":null,"id":"35fynswfx8p5","incident_id":"x3rdnh5mqb4x","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"operational","new_status":"operational"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"operational"},{"code":"fr85x46mwqgg","name":"Apps - Checkout","old_status":"operational","new_status":"operational"}]},{"status":"scheduled","body":"Please note that our APIs could be partially unavailable for two intervals of a few seconds each, from 08:00 am to 10:00 am CEST time on April 8 and 9, 2024. This maintenance is to ensure that our infrastructure is updated with the latest security measures. During this time, you may get errors while trying to call our APIs. Async requests will not be affected.","created_at":"2024-04-08T10:58:16.908+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-08T10:58:16.908+02:00","display_at":"2024-04-08T10:58:16.908+02:00","deliver_notifications":false,"tweet_id":null,"id":"4mv9n6qdbrqt","incident_id":"x3rdnh5mqb4x","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"operational","new_status":"operational"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"operational"},{"code":"fr85x46mwqgg","name":"Apps - Checkout","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Commerce API","created_at":"2018-05-03T07:43:20.605+02:00","updated_at":"2026-09-28T07:39:20.756+02:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"34km6s8trv84","page_id":"5sw7z2ldf26r","group_id":null},{"status":"operational","name":"Checkout","created_at":"2023-03-09T11:38:52.815+01:00","updated_at":"2024-06-15T05:26:42.126+02:00","position":1,"description":null,"showcase":true,"start_date":"2022-10-01T00:00:00.000+02:00","id":"fr85x46mwqgg","page_id":"5sw7z2ldf26r","group_id":"g5r1nct65t0v"},{"status":"operational","name":"Dashboard","created_at":"2018-05-03T07:45:17.394+02:00","updated_at":"2026-09-28T07:39:20.778+02:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"md7hnkpmydpq","page_id":"5sw7z2ldf26r","group_id":null}]},{"name":"Scheduled maintenance planned for April 8 and 9, 2024","status":"completed","created_at":"2024-03-14T11:27:14.174+01:00","updated_at":"2024-04-08T10:06:55.349+02:00","monitoring_at":null,"resolved_at":"2024-04-08T10:06:55.330+02:00","impact":"maintenance","shortlink":"https://stspg.io/yls7fdwmby06","scheduled_for":"2024-04-08T08:00:00.000+02:00","scheduled_until":"2024-04-08T10:00:00.000+02:00","scheduled_remind_prior":true,"scheduled_reminded_at":"2024-04-08T07:00:10.367+02:00","impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-03-14T11:27:14.165+01:00","reminder_intervals":null,"id":"x2jflyn7gnml","page_id":"5sw7z2ldf26r","incident_updates":[{"status":"completed","body":"The upgrade is now successfully completed. Our APIs are fully operational.","created_at":"2024-04-08T10:06:55.330+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-08T10:06:55.330+02:00","display_at":"2024-04-08T10:06:55.330+02:00","deliver_notifications":false,"tweet_id":null,"id":"kp56kdctb3ff","incident_id":"x2jflyn7gnml","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"operational","new_status":"operational"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"operational"},{"code":"fr85x46mwqgg","name":"Apps - Checkout","old_status":"operational","new_status":"operational"}]},{"status":"in_progress","body":"Please note that our APIs could be partially unavailable for two intervals of a few seconds in the upcoming couple of hours. This maintenance is to ensure that our infrastructure is updated with the latest security measures. During this time, you may get errors while trying to call our APIs. Async requests will not be affected.","created_at":"2024-04-08T08:07:36.698+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-08T08:07:36.698+02:00","display_at":"2024-04-08T08:07:36.698+02:00","deliver_notifications":false,"tweet_id":null,"id":"gmwsqtg1g2sr","incident_id":"x2jflyn7gnml","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"operational","new_status":"operational"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"operational"},{"code":"fr85x46mwqgg","name":"Apps - Checkout","old_status":"operational","new_status":"operational"}]},{"status":"scheduled","body":"Maintenance will begin as scheduled in 60 minutes.","created_at":"2024-04-08T07:00:10.246+02:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-08T07:00:10.246+02:00","display_at":"2024-04-08T07:00:10.246+02:00","deliver_notifications":true,"tweet_id":null,"id":"756bn67w4mjx","incident_id":"x2jflyn7gnml","custom_tweet":null,"affected_components":null},{"status":"scheduled","body":"Please note that our APIs could be partially unavailable for two intervals of a few seconds each, from 08:00 am to 10:00 am CEST time on April 8 and 9, 2024. This maintenance is to ensure that our infrastructure is updated with the latest security measures. During this time, you may get errors while trying to call our APIs. Async requests will not be affected.","created_at":"2024-03-14T11:27:14.229+01:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-03-14T11:27:14.229+01:00","display_at":"2024-03-14T11:27:14.229+01:00","deliver_notifications":false,"tweet_id":null,"id":"xf3pp69g4c41","incident_id":"x2jflyn7gnml","custom_tweet":null,"affected_components":[{"code":"34km6s8trv84","name":"Commerce API","old_status":"operational","new_status":"operational"},{"code":"md7hnkpmydpq","name":"Dashboard","old_status":"operational","new_status":"operational"},{"code":"fr85x46mwqgg","name":"Apps - Checkout","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Commerce API","created_at":"2018-05-03T07:43:20.605+02:00","updated_at":"2026-09-28T07:39:20.756+02:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"34km6s8trv84","page_id":"5sw7z2ldf26r","group_id":null},{"status":"operational","name":"Checkout","created_at":"2023-03-09T11:38:52.815+01:00","updated_at":"2024-06-15T05:26:42.126+02:00","position":1,"description":null,"showcase":true,"start_date":"2022-10-01T00:00:00.000+02:00","id":"fr85x46mwqgg","page_id":"5sw7z2ldf26r","group_id":"g5r1nct65t0v"},{"status":"operational","name":"Dashboard","created_at":"2018-05-03T07:45:17.394+02:00","updated_at":"2026-09-28T07:39:20.778+02:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"md7hnkpmydpq","page_id":"5sw7z2ldf26r","group_id":null}]}]
}