From 5a3754a53f1c1d2698a420d5fe565b89678476f8 Mon Sep 17 00:00:00 2001 From: Theo Arends <11044339+arendst@users.noreply.github.com> Date: Wed, 12 May 2021 13:03:20 +0200 Subject: [PATCH] Add OTA MQTT file upload --- tasmota/xdrv_02_mqtt.ino | 45 +++++++++-- tools/mqtt-file/upload-ota.py | 140 ++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 8 deletions(-) create mode 100644 tools/mqtt-file/upload-ota.py diff --git a/tasmota/xdrv_02_mqtt.ino b/tasmota/xdrv_02_mqtt.ino index eb972ae33..1adbfb674 100644 --- a/tasmota/xdrv_02_mqtt.ino +++ b/tasmota/xdrv_02_mqtt.ino @@ -1473,7 +1473,12 @@ void CmndFileUpload(void) { // Init upload buffer Mqtt.file_buffer = nullptr; - if (UPL_SETTINGS == Mqtt.file_type) { + if (UPL_TASMOTA == Mqtt.file_type) { + if (Update.begin(Mqtt.file_size)) { + Mqtt.file_buffer = &Mqtt.file_id; // Dummy buffer + } + } + else if (UPL_SETTINGS == Mqtt.file_type) { if (SettingsConfigBackup()) { Mqtt.file_buffer = settings_buffer; } @@ -1488,13 +1493,17 @@ void CmndFileUpload(void) { Mqtt.md5 = MD5Builder(); Mqtt.md5.begin(); -// TasmotaGlobal.masterlog_level = LOG_LEVEL_DEBUG_MORE; // Hide upload data logging + ResponseCmndChar(PSTR(D_JSON_STARTED)); + MqttPublishPrefixTopic_P(STAT, XdrvMailbox.command); // Enforce stat/wemos10/FILEUPLOAD } } else if ((Mqtt.file_id > 0) && (Mqtt.file_id != rcv_id)) { // Error receiving data - if (UPL_SETTINGS == Mqtt.file_type) { + if (UPL_TASMOTA == Mqtt.file_type) { + Update.end(true); + } + else if (UPL_SETTINGS == Mqtt.file_type) { SettingsBufferFree(); } @@ -1511,14 +1520,27 @@ void CmndFileUpload(void) { uint32_t bytes_left = Mqtt.file_size - Mqtt.file_pos; uint32_t read_bytes = (bytes_left < rcvd_bytes) ? bytes_left : rcvd_bytes; - uint8_t* buffer = Mqtt.file_buffer + Mqtt.file_pos; - memcpy(buffer, decode_output, read_bytes); - Mqtt.md5.add(buffer, read_bytes); + Mqtt.md5.add(decode_output, read_bytes); + + if (UPL_TASMOTA == Mqtt.file_type) { + Update.write(decode_output, read_bytes); + } else { + uint8_t* buffer = Mqtt.file_buffer + Mqtt.file_pos; + memcpy(buffer, decode_output, read_bytes); + } Mqtt.file_pos += read_bytes; + + if ((Mqtt.file_pos > rcvd_bytes) && ((Mqtt.file_pos % 102400) <= rcvd_bytes)) { + TasmotaGlobal.masterlog_level = LOG_LEVEL_NONE; // Enable logging + AddLog(LOG_LEVEL_DEBUG, PSTR(D_LOG_UPLOAD "Progress %d kB"), Mqtt.file_pos / 1024); + TasmotaGlobal.masterlog_level = LOG_LEVEL_DEBUG_MORE; // Hide upload data logging + } } if ((Mqtt.file_pos < Mqtt.file_size) || (Mqtt.file_md5.length() != 32)) { + TasmotaGlobal.masterlog_level = LOG_LEVEL_DEBUG_MORE; // Hide upload data logging + // {"Id":116,"MaxSize":"765"} Response_P(PSTR("{\"Id\":%d,\"MaxSize\":%d}"), Mqtt.file_id, FileUploadChunckSize()); } else { @@ -1529,7 +1551,14 @@ void CmndFileUpload(void) { // Process upload data en free buffer ResponseCmndDone(); - if (UPL_SETTINGS == Mqtt.file_type) { + if (UPL_TASMOTA == Mqtt.file_type) { + if (!Update.end(true)) { + ResponseCmndFailed(); + } else { + TasmotaGlobal.restart_flag = 2; // Always restart to re-enable disabled features during update + } + } + else if (UPL_SETTINGS == Mqtt.file_type) { if (!SettingsConfigRestore()) { ResponseCmndFailed(); } else { @@ -1543,7 +1572,7 @@ void CmndFileUpload(void) { } if (!Mqtt.file_buffer) { -// TasmotaGlobal.masterlog_level = LOG_LEVEL_NONE; // Enable logging + TasmotaGlobal.masterlog_level = LOG_LEVEL_NONE; // Enable logging Mqtt.file_id = 0; Mqtt.file_size = 0; Mqtt.file_type = 0; diff --git a/tools/mqtt-file/upload-ota.py b/tools/mqtt-file/upload-ota.py new file mode 100644 index 000000000..dbefcd69e --- /dev/null +++ b/tools/mqtt-file/upload-ota.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + upload-ota.py - Upload Tasmota firmware file + + Copyright (C) 2021 Theo Arends + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Requirements: + - Python 3.x and Pip: + sudo apt-get install python3 python3-pip + pip3 install paho-mqtt json + +Instructions: + Edit file and change parameters in User Configuration Section + + Then execute command upload-ota.py + +""" + +import paho.mqtt.client as mqtt +import time +import base64 +import hashlib +import json + +# **** Start of User Configuration Section + +broker = "domus1" # MQTT broker ip address or name +broker_port = 1883 # MQTT broker port + +mytopic = "demo" # Tasmota MQTT topic +#myfile = "../../build_output/firmware/tasmota32.bin" # Tasmota esp32 firmware file name +myfile = "../../build_output/firmware/tasmota.bin.gz" # Tasmota esp8266 firmware file name +myfiletype = 1 # Tasmota firmware file type + +# **** End of User Configuration Section + +# Derive fulltopic from broker LWT message +mypublish = "cmnd/"+mytopic+"/fileupload" +mysubscribe = "stat/"+mytopic+"/FILEUPLOAD" # Case sensitive + +Ack_flag = False + +file_id = 114 # Even id between 2 and 254 +file_chunk_size = 700 # Default Tasmota MQTT max message size + +# The callback for when mysubscribe message is received +def on_message(client, userdata, msg): + global Ack_flag + global file_chunk_size + + rcv_code = "" + rcv_id = 0 + +# print("Received message =",str(msg.payload.decode("utf-8"))) + + root = json.loads(msg.payload.decode("utf-8")) + if "Command" in root: rcv_code = root["Command"] + if rcv_code == "Error": + print("Error: Command error") + return + + if "Id" in root: rcv_id = root["Id"] + if rcv_id == file_id: + if "MaxSize" in root: file_chunk_size = root["MaxSize"] + + Ack_flag = False + +def wait_for_ack(): + global Ack_flag + timeout = 100 + while Ack_flag and timeout > 0: + time.sleep(0.01) + timeout = timeout -1 + + if Ack_flag: + print("Error: Ack timeout") + + return Ack_flag + +client = mqtt.Client() +client.on_message = on_message +client.connect(broker, broker_port) +client.loop_start() # Start loop to process received messages +client.subscribe(mysubscribe) + +time_start = time.time() +print("Uploading file "+myfile+" to "+mytopic+" ...") + +fo = open(myfile,"rb") +fo.seek(0, 2) # os.SEEK_END +file_size = fo.tell() +fo.seek(0, 0) # os.SEEK_SET + +client.publish(mypublish, "{\"File\":\""+myfile+"\",\"Id\":"+str("%3d"%file_id)+",\"Type\":"+str(myfiletype)+",\"Size\":"+str(file_size)+"}") +Ack_flag = True + +out_hash_md5 = hashlib.md5() + +Run_flag = True +while Run_flag: + if wait_for_ack(): # We use Ack here + Run_flag = False + + else: + chunk = fo.read(file_chunk_size) + if chunk: + out_hash_md5.update(chunk) # Update hash + base64_encoded_data = base64.b64encode(chunk) + base64_data = base64_encoded_data.decode('utf-8') + # Message length used by Tasmota (FileTransferHeaderSize) + client.publish(mypublish, "{\"Id\":"+str("%3d"%file_id)+",\"Data\":\""+base64_data+"\"}") + Ack_flag = True + + else: + md5_hash = out_hash_md5.hexdigest() + client.publish(mypublish, "{\"Id\":"+str("%3d"%file_id)+",\"Md5\":\""+md5_hash+"\"}") + Run_flag = False + +fo.close() + +time_taken = time.time() - time_start +print("Done in "+str("%.2f"%time_taken)+" seconds") + +client.disconnect() # Disconnect +client.loop_stop() # Stop loop