Recently I had solar panels installed on my home. There are tomes of information written about the subject of solar. The fact is that for my area, in 8 years I'll break even and then I have 17-25 years of the majority of my electrical generation take care of. But that isn't what this post is about.
SolarEdge, the company that produces the Power Optimizers and Inverter, provides a pretty nice web interface to monitor a whole slew of data down to individual panel power generation. (link) That's great, I love it, but I already have a bunch of graphs and such on my desktop that tracks various metrics for my desktop machine. This is mostly provided via Aquaero + Aquasuite by AquaComputer. I also use Rainmeter for a few things like disk and network utilization.
I figured adding some near real time data from the solar install to the desktop was a natural extension of my obsession with data. My first inclination was to approach it from trying to see if I could access the data directly from the inverter itself. I fiddle around with nmap and packet sniffing and was able to isolate the data transmission from the inverter up to the server. Unfortunately it is either encrypted or at the very least encoded somehow. Now I could spend a bunch of time reverse-engineering it, but the reality is I don't *need* real time data, just near real time. Based on my observations of the tracking site and the packet sniffing, it appears that the data is shipped over every minute and then summarized every 15 minutes. So the site would be a good place to get this information if it is in a relatively easy format to parse.
Firing up Chrome's developer tools I checked out what was being sent to the browser. Looking through the various calls I found this: https://monitoring.solaredge.com/solaredge-web/p/public_dashboard_data?fieldId=111098 (well, its slightly different for the logged in version, but close enough). Going here gives me a nice pretty JSON full of data to use.
{
"sumData":{
"siteName":"Wyza",
"siteInstallationDate":"04/02/2015",
"sitePeakPower":"11.34",
"siteAddress":"",
"siteCountry":"United States",
"siteState":"Indiana",
"siteUpdateDate":"04/07/2015 20:21",
"maxSeverity":"0",
"status":"Active"
},
"instData":{
"installerImage":"Morton Solar Logo 1.jpg",
"installerImagePath":"/installerImage?fieldId=123456",
"installerImageHash":"565362791",
useDefault:true
},
"imgData":{
"fieldId":"123456",
"image":"house.jpg",
"imagePath":"/fieldImage?fieldId=123456",
"imageHash":"277481497"
},
"overviewData":{
"lastDayEnergy":"21.82 kWh",
"lastMonthEnergy":"214.35 kWh",
"currentPower":"0 W",
"lifeTimeEnergy":"215.24 kWh"
,currencyCode:'USD'
,revenue:'35.16975'
,formatedRevenue:'$35.17'
},
"savingData":{
"CO2EmissionSaved":"326.91 lb",
"treesEquivalentSaved":"8.39",
"powerSavedInLightBulbs":"652.23"
},
powerChartData:
{
"start_week":1427846400000,
"energy_chart_month":[
{"name":2015,"data":[0,0,0,214.349,0,0,0,0,0,0,0,0]}
],
"energy_chart_month_max":214.349,
"month_uom":"kilo",
"energy_chart_quarter":[
{"name":2015,"data":[0,214.349,0,0]}
],
"quarter_uom":"kilo",
"year_categories":[2015],
"energy_chart_year":[
{"name":2015,"data":[214.349]}
],
"year_uom":"kilo"
},
energyChartData:
{
"field_start_date":{"year":2015,"month":3,"day":1},
"field_end_date":{"year":2015,"month":3,"day":7},
"year_range":[[2015]],
start_week:1427846400000,
GMTOffset:new Date(1427846400000).getTimezoneOffset()*60000,
end_week:{"year":2015,"month":3,"day":7},
end_week_next:1428451140000,
power_chart_week:[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,0,4.365,4.343,1.238,1.256,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,0,1.541,1.612,null,1.549,1.272,0.674,0.74,0.827,0.67,1.154,1.533,1.184,0.971,2.507,8.415,7.303,6.05,4.905,1.791,1.965,2.847,4.393,6.789,7.169,6.385,3.455,5.754,6.331,5.411,2.899,2.95,0.776,0.185,0.042,0.354,1.63,1.166,0.311,0.015,0,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,0.019,0.098,0.122,0.134,0.184,0.001,0.011,0.005,0.005,0,0.031,0.108,0.278,1.098,3.151,4.41,4.325,3.738,3.977,3.522,2.117,0.589,0.472,0.12,0.313,0.567,0.579,2.35,1.748,1.045,0.579,1.479,0.555,0.734,1.095,0.912,1.2,1.416,1.087,0.826,0.657,0.509,0.449,0.588,0.668,0.575,0.373,0.262,0.055,0,0,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,0,0.063,0.376,0.776,1.241,1.8,2.468,3.066,3.668,4.286,5.005,5.637,6.141,6.511,6.961,7.334,7.662,8.046,8.346,8.686,9.132,9.152,9.082,9.435,9.42,9.487,9.409,9.16,9.04,9.13,9.024,8.915,8.62,8.501,8.208,7.726,7.36,7.001,6.571,6.117,5.634,5.093,4.582,4.044,3.443,2.84,2.267,1.682,1.207,0.755,0.385,0.047,0,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,0,0.06,0.424,0.786,1.428,2.082,3.01,3.552,2.813,2.197,4.68,4.767,5.177,5.542,6.977,7.008,7.55,8.097,8.759,8.379,8.9,8.642,8.574,8.577,7.476,8.035,6.327,6.769,5.678,5.935,7.512,6.351,5.577,4.938,5.862,4.383,4.004,3.382,2.967,2.663,2.467,2.145,2.038,2.022,1.797,1.267,1.07,0.69,0.534,0.297,0.171,0.007,0,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,0,0,0.146,0.148,0.308,0.326,0.505,0.416,0.491,0.615,0.797,1.101,0.987,0.822,1.224,1.108,2.617,2.912,4.292,4.527,2.993,1.237,0.824,1.961,3.207,2.754,2.621,2.482,3.352,3.102,2.86,1.692,1.034,1.082,2.486,1.008,1.072,1.13,1.213,1.287,2.003,1.799,1.587,0.995,0.799,0.481,0.29,0.415,0.352,0.15,0.014,0,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,0,0,0.168,0.171,0.733,0.811,1.214,3.071,2.887,2.255,3.125,4.192,4.068,6.575,5.564,3.95,2.796,3.44,4.236,4.404,2.721,3.026,2.031,2.125,2.036,2.342,4.584,4.414,3.426,1.625,0.81,0.292,null,null,null,null,null,0,0.706,0.153,0.08,0.014,0.027,0.163,0.305,0.204,0.3,0.443,0.438,0.134,0.006,0,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],
power_chart_week_c:[],
power_chart_week_sc:[],
"energy_chart_month_by_day":{
"production":{"total":"214.349 kWh","roundValueTotal":"214.3489990234375","data":[2.345,26.07,14.175,74.674,56.923,18.344,21.818,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}
},
"energy_chart_month_by_day_total":214.3489990234375,
"energy_chart_month_by_day_max":74.674,
"energy_chart_month_by_day_total_c":0,
"energy_chart_month_by_day_max_c":0,
"energy_chart_month_by_day_total_sc":0,
"energy_chart_month_by_day_max_sc":0,
"energy_chart_year_by_month":{
"production":{"total":"214.349 kWh","roundValueTotal":"214.3489990234375","data":[0,0,0,214.349,0,0,0,0,0,0,0,0]}
},
"energy_chart_year_by_month_total":214.3489990234375,
"energy_chart_year_by_month_max":214.349,
"energy_chart_year_by_month_total_c":0,
"energy_chart_year_by_month_max_c":0,
"energy_chart_year_by_month_total_sc":0,
"energy_chart_year_by_month_max_sc":0,
"prev_month_date":1427899550000,
"next_month_date":1428451199000,
"prev_year_date":1427899550000,
"next_year_date":1428451199000,
"year_uom":"kilo",
"month_uom":"kilo" },
performanceRatioData:
{
fieldHasPR:false
}
}So yay, this is good, lots of tasty data to use. Well almost. The JSON does not properly conform to JSON standards. Some of the field keys aren't surrounded by quotes, some of them have single quotes, but the worst part was this line:
GMTOffset:new Date(1427846400000).getTimezoneOffset()*60000,JSON parsers really don't like this by default. But at this point I wasn't really sure if that was going to be a problem or not. My next thought was going to be how to take this data and render it to the desktop. (and then after determining that I could see how much the JSON formatting was a problem). Initially I had the thought to use Rainmeter. I was already using it for other collections and this seemed like a good fit. I dug into the plugin development area of rainmeter and for a while it seemed like I could make it work. But I found the biggest problem is that in the graph area it really didn't have a good way to provide historical data, which I really wanted to show. It mostly gave a method to update the graph every x seconds with a new point of data, but no way to pre-populate said graph. So that line of attack was a bust.
My next line of thinking was to write a Windows application that would stick to the desktop and provide the data. This does have some promise, but some folks have issue keeping it stuck at the bottom where it belongs. After mucking around for an hour or so, I really wasn't making progress and the day was wearing on.
My final line of thinking is where I ended up, which is to leverage Windows gadgets. Gadgets seemed fairly easy to code since they are basically HTML. The biggest downside here was that the renderer used is IE7. (Seriously MS? Would it have killed you to patch in the option to use at least IE10? I mean IE is terrible, but 10+ was less so.) There were other issues with using gagdets, chiefly that the data coming from JSON meant I was doing a cross domain request. (from localhost to monitoring.solaredge.com). Now you can get around this with jsonp *mostly*. But to get the data from being logged in, I needed to make a json request and then make another request with the same cookies and this is somewhat more difficult. The simple fact is that these constraints with cross domain AJAX requests are there for a reason and are not designed to be easily circumnavigated. The other problem was that the JSON was not quite what the browser would like (see above).
So the first thing I'd need to code is a proxy service. This will run locally on the machine and make the requests to the server for the data, clean it up so that it conforms, and then spit it back out on request. To do that I wrote a C# Windows Form that hides itself on startup and sits in the taskbar. There were 3 main classes that would handle this work. First I cleaned up the JSON data manually and used this tool to generate a C# object tree: http://json2csharp.com/. This resulted in the following class:
using System.Collections.Generic;
namespace SolarEdgeParser
{
public class SumData
{
public string siteName { get; set; }
public string siteInstallationDate { get; set; }
public string sitePeakPower { get; set; }
public string siteAddress { get; set; }
public string siteCountry { get; set; }
public string siteState { get; set; }
public string siteUpdateDate { get; set; }
public string maxSeverity { get; set; }
public string status { get; set; }
}
public class InstData
{
public string installerImage { get; set; }
public string installerImagePath { get; set; }
public string installerImageHash { get; set; }
public bool useDefault { get; set; }
}
public class ImgData
{
public string fieldId { get; set; }
public string image { get; set; }
public string imagePath { get; set; }
public string imageHash { get; set; }
}
public class OverviewData
{
public string lastDayEnergy { get; set; }
public string lastMonthEnergy { get; set; }
public string currentPower { get; set; }
public string lifeTimeEnergy { get; set; }
public string currencyCode { get; set; }
public string revenue { get; set; }
public string formatedRevenue { get; set; }
}
public class SavingData
{
public string CO2EmissionSaved { get; set; }
public string treesEquivalentSaved { get; set; }
public string powerSavedInLightBulbs { get; set; }
}
public class EnergyChartMonth
{
public int name { get; set; }
public List<double> data { get; set; }
}
public class EnergyChartQuarter
{
public int name { get; set; }
public List<double> data { get; set; }
}
public class EnergyChartYear
{
public int name { get; set; }
public List<double> data { get; set; }
}
public class PowerChartData
{
public long start_week { get; set; }
public List<EnergyChartMonth> energy_chart_month { get; set; }
public double energy_chart_month_max { get; set; }
public string month_uom { get; set; }
public List<EnergyChartQuarter> energy_chart_quarter { get; set; }
public string quarter_uom { get; set; }
public List<int> year_categories { get; set; }
public List<EnergyChartYear> energy_chart_year { get; set; }
public string year_uom { get; set; }
}
public class FieldStartDate
{
public int year { get; set; }
public int month { get; set; }
public int day { get; set; }
}
public class FieldEndDate
{
public int year { get; set; }
public int month { get; set; }
public int day { get; set; }
}
public class EndWeek
{
public int year { get; set; }
public int month { get; set; }
public int day { get; set; }
}
public class Production
{
public string total { get; set; }
public string roundValueTotal { get; set; }
public List<double> data { get; set; }
}
public class EnergyChartMonthByDay
{
public Production production { get; set; }
}
public class Production2
{
public string total { get; set; }
public string roundValueTotal { get; set; }
public List<double> data { get; set; }
}
public class EnergyChartYearByMonth
{
public Production2 production { get; set; }
}
public class EnergyChartData
{
public FieldStartDate field_start_date { get; set; }
public FieldEndDate field_end_date { get; set; }
public List<List<int>> year_range { get; set; }
public long start_week { get; set; }
public string GMTOffset { get; set; }
public EndWeek end_week { get; set; }
public long end_week_next { get; set; }
public List<double?> power_chart_week { get; set; }
public List<object> power_chart_week_c { get; set; }
public List<object> power_chart_week_sc { get; set; }
public EnergyChartMonthByDay energy_chart_month_by_day { get; set; }
public double energy_chart_month_by_day_total { get; set; }
public double energy_chart_month_by_day_max { get; set; }
public int energy_chart_month_by_day_total_c { get; set; }
public int energy_chart_month_by_day_max_c { get; set; }
public int energy_chart_month_by_day_total_sc { get; set; }
public int energy_chart_month_by_day_max_sc { get; set; }
public EnergyChartYearByMonth energy_chart_year_by_month { get; set; }
public double energy_chart_year_by_month_total { get; set; }
public double energy_chart_year_by_month_max { get; set; }
public int energy_chart_year_by_month_total_c { get; set; }
public int energy_chart_year_by_month_max_c { get; set; }
public int energy_chart_year_by_month_total_sc { get; set; }
public int energy_chart_year_by_month_max_sc { get; set; }
public long prev_month_date { get; set; }
public long next_month_date { get; set; }
public long prev_year_date { get; set; }
public long next_year_date { get; set; }
public string year_uom { get; set; }
public string month_uom { get; set; }
}
public class PerformanceRatioData
{
public bool fieldHasPR { get; set; }
}
public class SolarEdgeData
{
public SumData sumData { get; set; }
public InstData instData { get; set; }
public ImgData imgData { get; set; }
public OverviewData overviewData { get; set; }
public SavingData savingData { get; set; }
public PowerChartData powerChartData { get; set; }
public EnergyChartData energyChartData { get; set; }
public PerformanceRatioData performanceRatioData { get; set; }
}
}
Next we need to do the work to fetch the data, deserialize it into a POCO (Plain Old C# Object), and then re-serialize it. I do this so that it will return a nice clean well formatted JSON. To fetch it we need to first login to get a session cookie. Then we'll use that to go fetch the data.
using System;
using System.Net;
using System.Text.RegularExpressions;
using System.IO;
using System.Web.Script.Serialization;
namespace SolarEdgeParser
{
public class SolarEdge
{
public string GetSerializedData()
{
var cookiejar = new CookieContainer();
Login(ref cookiejar);
var request = (HttpWebRequest)WebRequest.Create("https://monitoring.solaredge.com/solaredge-web/p/dashboard_data?fieldId=111098");
request.CookieContainer = cookiejar;
var response = (HttpWebResponse)request.GetResponse();
var data = response.GetResponseStream();
var reader = new StreamReader(data);
var result = reader.ReadToEnd();
result = Regex.Replace(result, @"GMTOffset:new Date\(\d+\)\.getTimezoneOffset\(\)\*60000", "GMTOffset: " + TimeZoneInfo.Local.GetUtcOffset(DateTime.Now).Hours);
var parsedData = new JavaScriptSerializer().Deserialize<SolarEdgeData>(result);
reader.Close();
data.Close();
response.Close();
return new JavaScriptSerializer().Serialize(parsedData);
}
void Login(ref CookieContainer cookiejar)
{
var request = (HttpWebRequest)WebRequest.Create("https://monitoring.solaredge.com/solaredge-web/p/submitLogin");
request.CookieContainer = cookiejar;
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
using (var writer = new StreamWriter(request.GetRequestStream()))
{
writer.Write("cmd=login&demo=false&username=nope&password=notachance");
writer.Flush();
}
var response = (HttpWebResponse)request.GetResponse();
var data = response.GetResponseStream();
var reader = new StreamReader(data);
var result = reader.ReadToEnd();
reader.Close();
data.Close();
response.Close();
if (Regex.Match(result, "Failure", RegexOptions.IgnoreCase).Success)
{
throw new Exception("Login Failure");
}
}
}
}
using System;
using System.Windows.Forms;
using System.Net;
using SolarEdgeParser;
namespace SolarEdgeProxy
{
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
notifyIcon1.Visible = true;
var service = new SolarEdgeProxyService();
this.Hide();
}
public class SolarEdgeProxyService
{
private Listener listener;
public SolarEdgeProxyService()
{
listener = new HttpListener();
listener.Prefixes.Add("http://localhost:8001/");
listener.Start();
IAsyncResult result = listener.BeginGetContext(new AsyncCallback(ListenerCallback), listener);
}
public static void ListenerCallback(IAsyncResult result)
{
HttpListener listener = (HttpListener)result.AsyncState;
// Call EndGetContext to complete the asynchronous operation.
HttpListenerContext context = listener.EndGetContext(result);
HttpListenerRequest request = context.Request;
// Obtain a response object.
HttpListenerResponse response = context.Response;
// Construct a response.
var se = new SolarEdge();
var responseString = se.GetSerializedData();
byte[] buffer = System.Text.Encoding.UTF8.GetBytes(responseString);
// Get a response stream and write the response to it.
response.ContentLength64 = buffer.Length;
System.IO.Stream output = response.OutputStream;
output.Write(buffer, 0, buffer.Length);
// You must close the output stream.
output.Close();
listener.BeginGetContext(new AsyncCallback(ListenerCallback), listener);
}
}
private void Form1_Resize(object sender, EventArgs e)
{
this.Hide();
}
private void notifyIcon1_MouseDoubleClick(object sender, MouseEventArgs e)
{
this.Show();
}
}
}
No comments:
Post a Comment