added core dump upload support

master
Jan Umbach 2023-08-20 18:03:51 +02:00
parent 479fea5331
commit ab80aebf04
10 changed files with 452 additions and 13 deletions

View File

@ -23,4 +23,6 @@ idf_component_register(
set_target_properties(${COMPONENT_LIB} PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/include"
)
)
idf_build_set_property(LINK_OPTIONS "-Wl,--wrap=esp_panic_handler" APPEND)

View File

@ -71,6 +71,13 @@ menu "Jannex Configuration"
help
The length of the debug cache. The cache is used to store the last debug chars. The cache is used to send the last debug chars to the client when the client connects to the debug server or when the client reconnects to the debug server.
config DEBUG_SEND_COREDUMP
bool "Send coredump to debug server"
default n
depends on HTTP_SERVER_ENABLE_DEBUG_MODE
help
Send coredump to debug server
config NTP_SERVER
string "NTP Server"
default "0.de.pool.ntp.org"

View File

@ -12,4 +12,6 @@
void initConsole();
void connectConsole();
void sendCoredumpToConsole();
#endif

View File

@ -0,0 +1,11 @@
#pragma once
#include <esp_http_server.h>
extern httpd_uri_t coreDumpRequest;
extern uint16_t panicRetryCounter;
extern uint16_t softwareResetCounter;
bool checkIfCoredumpWasDownloaded();
bool setCoredumpWasDownloaded(bool value);

View File

@ -16,8 +16,13 @@
#include <esp_log.h>
#include <esp_system.h>
#include "esp_app_desc.h"
#include "esp_mac.h"
#include "mbedtls/base64.h"
#include "esp_partition.h"
#define CALC_BASE64_ENCODED_LENGTH(input_length) ((4 * ((input_length + 2) / 3)) + 1)
char cacheMessage[CONFIG_HTTP_SERVER_DEBUG_CACHE_LENGTH];
@ -26,13 +31,105 @@ int cacheMessageIndex = 0;
SemaphoreHandle_t cacheMutex;
const static char *TAG = "Console";
const static char *LOG_TAG = "[LOG]";
const static char *TAG = "ConsoleWS";
esp_websocket_client_handle_t client = NULL;
bool shouldSendCoreDump = false;
int skippedMessages = 0;
static const size_t BLOCK_SIZE = 1024;
void sendCoreDump()
{
if (shouldSendCoreDump)
{
shouldSendCoreDump = false;
esp_websocket_client_send_text(client, "!DMSG_START!", 12, 10000 / portTICK_PERIOD_MS);
esp_partition_t *coredump_partition = esp_partition_find_first(
ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_COREDUMP, "coredump");
if (coredump_partition == NULL)
{
ESP_LOGE(TAG, "Core dump partition not found.");
goto END;
}
uint8_t *buffer = (uint8_t *)malloc(BLOCK_SIZE);
size_t remaining = coredump_partition->size;
// size_t base64_length = CALC_BASE64_ENCODED_LENGTH(BLOCK_SIZE) + 1;
// char *base64 = (char *)malloc(base64_length + 1 + 1); // + \n + \0
// size_t encoded_length = 0;
while (remaining > 0)
{
// wait until wifi is connected
// xEventGroupWaitBits(s_wifi_event_group, NETWORK_CONNECTED_BIT, false, true, portMAX_DELAY);
// wait until console is connected
if (client == NULL || !esp_websocket_client_is_connected(client))
{
vTaskDelay(1000 / portTICK_PERIOD_MS);
continue;
}
size_t to_read = (remaining < BLOCK_SIZE) ? remaining : BLOCK_SIZE;
esp_err_t err = esp_partition_read(coredump_partition, coredump_partition->size - remaining, buffer, to_read);
if (err != ESP_OK)
{
ESP_LOGE(TAG, "Failed to read core dump data.");
free(buffer);
// free(base64);
goto END;
}
/*if (base64 != NULL)
{
int ret = mbedtls_base64_encode((unsigned char *)base64, base64_length - 1, &encoded_length, (unsigned char *)buffer, to_read);
if (ret != 0)
{
ESP_LOGE(TAG, "mbedtls_base64_encode failed!");
free(buffer);
free(base64);
goto END;
}
// set last char to \n
base64[encoded_length] = '\n';
// base64[encoded_length + 1] = '\0';
}
else
{
ESP_LOGE(TAG, "Memory allocation failed!");
free(buffer);
free(base64);
goto END;
}*/
esp_websocket_client_send_bin(client, (const char *)buffer, to_read, 10000 / portTICK_PERIOD_MS);
remaining -= to_read;
// vTaskDelay(1);
printf("free heap: %lu\n", esp_get_free_heap_size());
}
esp_websocket_client_send_text(client, "!DMSG_END!", 10, 10000 / portTICK_PERIOD_MS);
free(buffer);
// free(base64);
}
END:
}
void sendOlderMessages()
{
if (esp_websocket_client_is_connected(client))
@ -225,6 +322,10 @@ static void websocket_event_handler(void *handler_args, esp_event_base_t base, i
case WEBSOCKET_EVENT_CONNECTED:
ESP_LOGI(TAG, "WEBSOCKET_EVENT_CONNECTED");
sendOlderMessages();
printf("sending core dump\n");
sendCoreDump();
printf("sending core dump done\n");
// xTaskCreate(sendCoreDumpTask, "sendCoreDumpTask", 4096, NULL, 5, NULL);
ESP_LOGI(TAG, "Websocket connected! %s", CONFIG_HTTP_SERVER_DEBUG_URL);
break;
@ -245,7 +346,6 @@ static void websocket_event_handler(void *handler_args, esp_event_base_t base, i
break;
}
ESP_LOGI(TAG, "WEBSOCKET_EVENT_DATA");
ESP_LOGI(TAG, "Received opcode=%d", data->op_code);
if (data->op_code == 0x08 && data->data_len == 2)
{
@ -253,11 +353,37 @@ static void websocket_event_handler(void *handler_args, esp_event_base_t base, i
}
else
{
ESP_LOGW(TAG, "Received=%.*s", data->data_len, (char *)data->data_ptr);
ESP_LOGI(TAG, "Received=%.*s", data->data_len, (char *)data->data_ptr);
// check if message is coredumpOK
if (data->data_len == 10 && strncmp((char *)data->data_ptr, "coredumpOK", 10) == 0)
{
// delete coredump partition
esp_partition_t *coredump_partition = esp_partition_find_first(
ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_COREDUMP, "coredump");
if (coredump_partition == NULL)
{
ESP_LOGE(TAG, "Core dump partition not found.");
break;
}
esp_err_t err = esp_partition_erase_range(coredump_partition, 0, coredump_partition->size);
if (err != ESP_OK)
{
ESP_LOGE(TAG, "Failed to erase core dump partition.");
break;
}
// set coredump was downloaded
// setCoredumpWasDownloaded(true);
ESP_LOGI(TAG, "Core dump partition erased.");
}
}
// If received data contains json structure it succeed to parse
ESP_LOGW(TAG, "Total payload length=%d, data_len=%d, current payload offset=%d\r\n", data->payload_len, data->data_len, data->payload_offset);
// ESP_LOGW(TAG, "Total payload length=%d, data_len=%d, current payload offset=%d\r\n", data->payload_len, data->data_len, data->payload_offset);*/
break;
case WEBSOCKET_EVENT_ERROR:
@ -277,9 +403,25 @@ static void websocket_event_handler(void *handler_args, esp_event_base_t base, i
static void console_task(void *pvParameters)
{
// get mac address
uint8_t mac[6];
esp_efuse_mac_get_default(mac);
// get current version
const esp_app_desc_t *app_desc = esp_app_get_description();
char branch[32];
getBranch(branch);
char url[256];
sprintf(url, "%s?mac=%02X:%02X:%02X:%02X:%02X:%02X&device=%s&branch=%s&ver=%s", CONFIG_HTTP_SERVER_DEBUG_URL, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5], app_desc->project_name, branch, app_desc->version);
esp_websocket_client_config_t websocket_cfg = {
.uri = CONFIG_HTTP_SERVER_DEBUG_URL,
.uri = url,
.port = CONFIG_HTTP_SERVER_DEBUG_PORT,
.task_stack = 1024 * 6,
.reconnect_timeout_ms = 5000,
.network_timeout_ms = 500,
@ -309,6 +451,7 @@ static void console_task(void *pvParameters)
while (true)
{
sendOlderMessages();
vTaskDelay(500 / portTICK_PERIOD_MS);
}
}
@ -343,4 +486,9 @@ void connectConsole()
xTaskCreate(console_task, "console_task", 4096, NULL, 5, NULL);
}
#endif // CONFIG_HTTP_SERVER_ENABLE_DEBUG_MODE
void sendCoredumpToConsole()
{
shouldSendCoreDump = true;
}
#endif // CONFIG_HTTP_SERVER_ENABLE_DEBUG_MODE

80
src/api/coredump.c Normal file
View File

@ -0,0 +1,80 @@
#include "jannex/jannex.h"
#include <stdio.h>
#include <string.h>
#include "esp_log.h"
#include "esp_partition.h"
#include "esp_http_server.h"
#include <esp_http_server.h>
static const char *TAG = "coredumpServer";
static const size_t BLOCK_SIZE = 1024; // Adjust the block size as needed
RTC_NOINIT_ATTR uint16_t panicRetryCounter;
RTC_NOINIT_ATTR uint16_t softwareResetCounter;
bool checkIfCoredumpWasDownloaded()
{
return false;
}
bool setCoredumpWasDownloaded(bool value)
{
return true;
}
esp_err_t coredump_get_handler(httpd_req_t *req)
{
esp_partition_t *coredump_partition = esp_partition_find_first(
ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_COREDUMP, "coredump");
if (coredump_partition == NULL)
{
ESP_LOGE(TAG, "Core dump partition not found.");
httpd_resp_send_err(req, HTTPD_500_INTERNAL_SERVER_ERROR, "Core dump not available");
return ESP_FAIL;
}
// Set the HTTP response headers for downloading a .bin file
httpd_resp_set_type(req, "application/octet-stream");
httpd_resp_set_hdr(req, "Content-Disposition", "attachment; filename=coredump.bin");
uint8_t *buffer = (uint8_t *)malloc(BLOCK_SIZE);
if (buffer == NULL)
{
ESP_LOGE(TAG, "Memory allocation failed.");
httpd_resp_send_err(req, HTTPD_500_INTERNAL_SERVER_ERROR, "Memory allocation failed");
return ESP_FAIL;
}
size_t remaining = coredump_partition->size;
while (remaining > 0)
{
size_t to_read = (remaining < BLOCK_SIZE) ? remaining : BLOCK_SIZE;
esp_err_t err = esp_partition_read(coredump_partition, coredump_partition->size - remaining, buffer, to_read);
if (err != ESP_OK)
{
ESP_LOGE(TAG, "Failed to read core dump data.");
free(buffer);
httpd_resp_send_err(req, HTTPD_500_INTERNAL_SERVER_ERROR, "Failed to read core dump data");
return ESP_FAIL;
}
// Send a block of data as the HTTP response
httpd_resp_send_chunk(req, (const char *)buffer, to_read);
remaining -= to_read;
}
free(buffer);
httpd_resp_send_chunk(req, NULL, 0); // Send an empty chunk to finish the response
return ESP_OK;
}
httpd_uri_t coreDumpRequest = {
.uri = "/getCoreDumpData",
.method = HTTP_GET,
.handler = coredump_get_handler,
.user_ctx = NULL};

View File

@ -2,6 +2,7 @@
#include "jannex/ota.h"
#include "jannex/api/console.h"
#include "jannex/api/coredump.h"
#include <string.h>
#include "esp_log.h"
@ -13,10 +14,21 @@
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_partition.h"
#include "esp_netif.h"
#include "esp_netif_sntp.h"
#include "esp_sntp.h"
#include "esp_private/panic_internal.h"
#include "esp_debug_helpers.h"
#include "esp_chip_info.h"
extern uint16_t panicRetryCounter; // RTC_NOINIT_ATTR
extern uint16_t softwareResetCounter; // RTC_NOINIT_ATTR
const char *reset_reason_to_string(esp_reset_reason_t reason);
const char *getChipModelString(esp_chip_model_t chip_model);
void time_sync_notification_cb(struct timeval *tv)
{
ESP_LOGI(jannexTAG, "Notification of a time synchronization event");
@ -81,6 +93,7 @@ void startJannex(void *parameter)
#endif // CONFIG_HTTP_SERVER_ENABLE_DEBUG_MODE
initialize_sntp();
vTaskDelete(NULL);
}
@ -95,12 +108,101 @@ void initJannex()
const esp_app_desc_t *app_desc = esp_app_get_description();
esp_chip_info_t chip_info;
esp_chip_info(&chip_info);
ESP_LOGI(jannexTAG, "Initializing Jannex");
ESP_LOGI(jannexTAG, "Current Build version: %s", app_desc->version);
ESP_LOGI(jannexTAG, "Current Build name: %s", app_desc->project_name);
ESP_LOGI(jannexTAG, "Current Build date: %s", app_desc->date);
ESP_LOGI(jannexTAG, "Current Build time: %s", app_desc->time);
bool coredumpAvailable = false;
// check for core dump from flash partition
esp_partition_t *coredump_partition = esp_partition_find_first(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_COREDUMP, "coredump");
uint8_t *buffer = (uint8_t *)malloc(coredump_partition->size);
esp_err_t err = esp_partition_read(coredump_partition, 0, buffer, coredump_partition->size);
if (err == ESP_OK)
{
bool cdAvailable = false;
// sum the first 100 bytes of buffer
for (int i = 0; i < 3000; i++)
{
if (buffer[i] != 0xFF)
{
cdAvailable = true;
break;
}
}
if (cdAvailable)
{
coredumpAvailable = true;
ESP_LOGE(jannexTAG, "---------- CORE DUMP INFO ----------");
}
}
else
{
ESP_LOGW(jannexTAG, "Coredump part read failed: %s", esp_err_to_name(err));
}
free(buffer);
esp_reset_reason_t reset_reason = esp_reset_reason();
ESP_LOGI(jannexTAG, "JNXJSON{\n"
" \"ver\": \"%s\",\n"
" \"chip\": \"%s\",\n"
" \"chipRev\": \"%i\",\n"
" \"name\": \"%s\",\n"
" \"build_date\": \"%s\",\n"
" \"build_time\": \"%s\",\n"
" \"rstRsn\": \"%s\",\n"
" \"softRst\": %i,\n"
" \"panicRst\": %i\n"
"}JNXJSON",
app_desc->version,
getChipModelString(chip_info.model),
chip_info.revision,
app_desc->project_name,
app_desc->date,
app_desc->time,
reset_reason_to_string(reset_reason),
softwareResetCounter,
panicRetryCounter);
/*ESP_LOGI(jannexTAG, "Reset: %s", reset_reason_to_string(reset_reason));
ESP_LOGI(jannexTAG, "Software resets: %i", softwareResetCounter);
ESP_LOGI(jannexTAG, "Panic retries: %i", panicRetryCounter);
ESP_LOGI(jannexTAG, "Build version: %s", app_desc->version);
ESP_LOGI(jannexTAG, "\" name: %s", app_desc->project_name);
ESP_LOGI(jannexTAG, "\" date: %s", app_desc->date);
ESP_LOGI(jannexTAG, "\" time: %s", app_desc->time);
ESP_LOGI(jannexTAG, "Chip model: %s", getChipModelString(chip_info.model));
ESP_LOGI(jannexTAG, "\" revision: %i", chip_info.revision);*/
if (reset_reason == ESP_RST_POWERON)
{
softwareResetCounter = 0;
panicRetryCounter = 0;
}
else
{
softwareResetCounter++;
}
if (coredumpAvailable)
{
ESP_LOGE(jannexTAG, "-------- CORE DUMP INFO END --------");
#ifdef CONFIG_DEBUG_SEND_COREDUMP
ESP_LOGI(jannexTAG, "Core dump available. Sending it to the server:");
sendCoredumpToConsole();
#else
ESP_LOGI(jannexTAG, "Core dump available. Please download it from the web interface.");
#endif
}
xTaskCreate(startJannex, "startJannex", 4096, NULL, 5, NULL);
xTaskCreate(checkIfOTAwasSuccessful, "checkIfOTAwasSuccessful", 4096, NULL, 5, NULL);
@ -109,4 +211,85 @@ void initJannex()
void updateJannex()
{
xTaskCreate(checkForUpdatesAndInstall, "checkForUpdatesAndInstall", 8192, NULL, 5, NULL);
}
// Declare the real panic handler function. We'll be able to call it after executing our custom code
void __real_esp_panic_handler(void *);
// This function will be considered the esp_panic_handler to call in case a panic occurs
void __wrap_esp_panic_handler(panic_info_t *info)
{
// Custom code, count the number of panics or simply print a message
panicRetryCounter++;
// Convert the integer to a string
char retryCounterStr[10]; // Adjust the size as needed
snprintf(retryCounterStr, sizeof(retryCounterStr), "%d", panicRetryCounter);
// Create the message with the retry counter value
char message[100]; // Adjust the size as needed
strcpy(message, "Panic has been triggered by the program! Retry counter: ");
strcat(message, retryCounterStr);
// Print the message using esp_rom_printf
esp_rom_printf(message);
// Call the original panic handler function to finish processing this error (creating a core dump for example...)
__real_esp_panic_handler(info);
}
const char *reset_reason_to_string(esp_reset_reason_t reason)
{
switch (reason)
{
case ESP_RST_UNKNOWN:
return "Unknown";
case ESP_RST_POWERON:
return "Power-on";
case ESP_RST_EXT:
return "External pin";
case ESP_RST_SW:
return "Software reset";
case ESP_RST_PANIC:
return "Panic reset";
case ESP_RST_INT_WDT:
return "Interrupt watchdog reset";
case ESP_RST_TASK_WDT:
return "Task watchdog reset";
case ESP_RST_WDT:
return "Other watchdog reset";
case ESP_RST_DEEPSLEEP:
return "Deep sleep reset";
case ESP_RST_BROWNOUT:
return "Brownout reset";
case ESP_RST_SDIO:
return "SDIO reset";
default:
return "Invalid reset reason value";
}
}
const char *getChipModelString(esp_chip_model_t chip_model)
{
switch (chip_model)
{
case CHIP_ESP32:
return "ESP32";
case CHIP_ESP32S2:
return "ESP32-S2";
case CHIP_ESP32S3:
return "ESP32-S3";
case CHIP_ESP32C3:
return "ESP32-C3";
case CHIP_ESP32C2:
return "ESP32-C2";
case CHIP_ESP32C6:
return "ESP32-C6";
case CHIP_ESP32H2:
return "ESP32-H2";
case CHIP_POSIX_LINUX:
return "POSIX/Linux Simulator";
default:
return "Unknown";
}
}

View File

@ -227,6 +227,7 @@ void printFreeHeapTask(void *parameter)
void checkForUpdatesAndInstall(void *parameter)
{
// ESP_ERROR_CHECK(heap_trace_init_standalone(trace_record, NUM_RECORDS));
xTaskCreate(printFreeHeapTask, "printFreeHeapTask", 4096, NULL, 5, NULL);

View File

@ -15,6 +15,8 @@
static const char *TAG = "otaUpgrade";
int ota_download_buffer_size = 4096 * 4;
esp_err_t _http_event_handler(esp_http_client_event_t *evt)
{
switch (evt->event_id)
@ -65,7 +67,7 @@ void startOTAupdate(bool forceUpdate)
esp_http_client_config_t config = {
.url = url,
.crt_bundle_attach = esp_crt_bundle_attach,
.buffer_size = 4096 * 4,
.buffer_size = ota_download_buffer_size,
.event_handler = _http_event_handler,
.keep_alive_enable = true,
@ -89,5 +91,6 @@ void startOTAupdate(bool forceUpdate)
else
{
ESP_LOGE(TAG, "Firmware upgrade failed!");
ota_download_buffer_size = 1024;
}
}

View File

@ -3,6 +3,7 @@
#ifdef WEB_SERVER_ENABLED
#include "jannex/api/customAPIHandler.h"
#include "jannex/api/coredump.h"
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
@ -120,6 +121,7 @@ static httpd_handle_t start_webserver(void)
// Set URI handlers
ESP_LOGI(TAG, "Registering URI handlers");
httpd_register_uri_handler(server, &hello);
httpd_register_uri_handler(server, &coreDumpRequest);
ESP_LOGI(TAG, "Registering custom URI handlers..");