Feature/cumulative graph

README.md

@@ -32,6 +32,9 @@ When representing the cost profile for individual resources, Ice will factor the
 5. Breakdown page of Application Groups
 ![Breakdown page of Application Groups](https://github.com/Netflix/ice/blob/master/screenshots/ss_breakdown_appgroup.png?raw=true)
 
+6. Estimate page with cumulative values
+![Estimate page using cumulative values](./screenshots/ss_estimate.png?raw=true)
+
 ##Prerequisite:
 
 1. First sign up for Amazon's programmatic billing access [here](http://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/detailed-billing-reports.html) to receive detailed billing(hourly) reports. Verify you receive monthly billing file in the following format: `<accountid>-aws-billing-detailed-line-items-<year>-<month>.csv.zip`.
@@ -169,7 +172,7 @@ Options with * require writing your own code.
           ice.reservationPeriod=threeyear
           # reservation utilization, possible values are LIGHT, HEAVY
           ice.reservationUtilization=HEAVY
-
+          
 2. Reservation capacity poller
 
   To use BasicReservationService, you should also run reservation capacity poller, which will call ec2 API (describeReservedInstances) to poll reservation capacities for each reservation owner account defined in ice.properties. The reservation capacities history is stored in a file in s3 bucket. To run reservation capacity poller, following steps below:
@@ -240,6 +243,25 @@ Options with * require writing your own code.
 
   You may also want to show your organization's throughput metric alongside usage and cost. You can choose to implement interface ThroughputMetricService, or you can simply use the existing BasicThroughputMetricService. Using BasicThroughputMetricService requires the throughput metric data to be stores monthly in files with names like <filePrefix>_2013_04, <filePrefix>_2013_05. Data in files should be delimited by new lines. <filePrefix> is specified when you create BasicThroughputMetricService instance.
 
+9. Estimate Page
+
+  You may want to use the estimate page to visualize estimated velocity against actual spending velocity.  To enable:
+
+	ice.report.estimate=true
+
+  For each account, you can set the estimate in the ice.properties:
+
+	ice.account.dailyestimate.account1=500
+	ice.account.dailyestimate.account2=505
+
+  Also supports historical estimates:
+
+	ice.account.dailyestimate.accountname.2015-01-26=500
+	ice.account.dailyestimate.accountname.2015-01-30=550
+	ice.account.dailyestimate.accountname.2015-02-15=650
+
+
+
 ##Support
 
 Please use the [Ice Google Group](https://groups.google.com/d/forum/iceusers) for general questions and discussion.

grails-app/conf/BootStrap.groovy

@@ -45,6 +45,9 @@ import com.netflix.ice.common.ProductService
 import com.netflix.ice.basic.BasicResourceService
 import com.netflix.ice.basic.BasicWeeklyCostEmailService
 import com.netflix.ice.reader.ApplicationGroupService
+import org.joda.time.format.ISODateTimeFormat
+import org.joda.time.format.DateTimeFormatter
+
 
 class BootStrap {
     private static boolean initialized = false;
@@ -102,11 +105,43 @@ class BootStrap {
 
             Map<String, Account> accounts = Maps.newHashMap();
             for (String name: prop.stringPropertyNames()) {
-                if (name.startsWith("ice.account.")) {
+                if (name.startsWith("ice.account.dailyestimate.")) {
+                    String propertyValue = prop.getProperty(name);
+                    Double propertyDoubleValue = Double.parseDouble(propertyValue);
+
+                    String propertyNameNoPrefix = name.substring("ice.account.dailyestimate.".length());
+                    String accountName=propertyNameNoPrefix;
+                    String[] propertySections = propertyNameNoPrefix.split("\\.");
+                    DateTime estimateDate = new DateTime(0);
+                    if (propertySections.length == 2) {
+                        DateTimeFormatter dtf = ISODateTimeFormat.date();
+                        accountName = propertySections[0];
+                        estimateDate = dtf.parseDateTime(propertySections[1]);
+                    }
+
+                    Account account = accounts.get(accountName);
+                    if (account == null) {
+                        String accountId = prop.getProperty("ice.account." + accountName);
+                        if (accountId == null) {
+                            System.err.println(accountName + " does not have an ice.account entry");
+                            continue;
+                        }
+                        account = new Account(accountId, accountName);
+                        accounts.put(accountName, account);
+                    }
+                    System.out.println("Set Daily Estimate for " + account + " - " + propertyDoubleValue.toString());
+                    account.dailyEstimates.put(estimateDate, propertyDoubleValue);
+                } else if (name.startsWith("ice.account.") ) {
                     String accountName = name.substring("ice.account.".length());
-                    accounts.put(accountName, new Account(prop.getProperty(name), accountName));
+                    Account account = accounts.get(accountName);
+                    if (account == null) {
+                        accounts.put(accountName, new Account(prop.getProperty(name), accountName));
+                    } else {
+                        //loaded above
+                    }
                 }
             }
+
             Map<Account, List<Account>> reservationAccounts = Maps.newHashMap();
             Map<Account, String> reservationAccessRoles = Maps.newHashMap();
             Map<Account, String> reservationAccessExternalIds = Maps.newHashMap();
@@ -198,6 +233,8 @@ class BootStrap {
                     properties.setProperty(IceOptions.CURRENCY_SIGN, prop.getProperty(IceOptions.CURRENCY_SIGN));
                 if (prop.getProperty(IceOptions.HIGHSTOCK_URL) != null)
                     properties.setProperty(IceOptions.HIGHSTOCK_URL, prop.getProperty(IceOptions.HIGHSTOCK_URL));
+                if (prop.getProperty(IceOptions.ESTIMATE_REPORT) != null)
+                    properties.setProperty(IceOptions.ESTIMATE_REPORT, prop.getProperty(IceOptions.ESTIMATE_REPORT));
 
                 ResourceService resourceService = StringUtils.isEmpty(properties.getProperty(IceOptions.CUSTOM_TAGS)) ? null : new BasicResourceService();
                 ApplicationGroupService applicationGroupService = new BasicS3ApplicationGroupService();

grails-app/controllers/com/netflix/ice/DashboardController.groovy

@@ -377,6 +377,8 @@ class DashboardController {
 
     def detail = {}
 
+    def estimates={}
+
     def reservation = {}
 
     def breakdown = {}
@@ -393,6 +395,10 @@ class DashboardController {
 
         TagType groupBy = query.getString("groupBy").equals("None") ? null : TagType.valueOf(query.getString("groupBy"));
         boolean isCost = query.getBoolean("isCost");
+        boolean includeEstimates = false;
+        if (query.has("includeEstimates")) {
+            includeEstimates = query.getBoolean("includeEstimates");
+        }
         boolean breakdown = query.getBoolean("breakdown");
         boolean showsps = query.getBoolean("showsps");
         boolean factorsps = query.getBoolean("factorsps");
@@ -438,6 +444,11 @@ class DashboardController {
         }
         interval = roundInterval(interval, consolidateType);
 
+        //we will have an estimate for each datapoint
+        Map<Tag, double[]> estimates;
+
+
+
         Map<Tag, double[]> data;
         if (groupBy == TagType.ApplicationGroup) {
             data = Maps.newTreeMap();
@@ -550,12 +561,22 @@ class DashboardController {
                 aggregate,
                 forReservation
             );
+            //groupBy Account can have estimates
+            if (includeEstimates) {
+                DataManager estimateManager = getManagers().getEstimateManager(consolidateType);
+                estimates = estimateManager.getData(interval, new TagLists(accounts, regions, zones, products, operations, usageTypes, resourceGroups), groupBy, aggregate, forReservation);
+            }
         }
-        def stats = getStats(data);
+        def stats = getStats(data, estimates);
         if (aggregate == AggregateType.stats && data.size() > 1)
             data.remove(Tag.aggregated);
 
         def result = [status: 200, start: interval.getStartMillis(), data: data, stats: stats, groupBy: groupBy == null ? "None" : groupBy.name()]
+        if (estimates != null)
+        {
+            result.put("estimates", estimates);
+        }
+
         if (breakdown && data.size() > 0 && data.values().iterator().next().length > 0) {
             result.time = new IntRange(0, data.values().iterator().next().length - 1).collect {
                 if (consolidateType == ConsolidateType.daily)
@@ -657,7 +678,7 @@ class DashboardController {
         }
     }
 
-    private Map<Tag, Map> getStats(Map<Tag, double[]> data) {
+    private Map<Tag, Map> getStats(Map<Tag, double[]> data, Map<Tag, double[]> estimates) {
         def result = [:];
 
         for (Map.Entry<Tag, double[]> entry: data.entrySet()) {
@@ -675,6 +696,15 @@ class DashboardController {
                 total += v;
             }
             result[tag] = [max: max, total: total, average: total / values.length];
+            double totalEstimate = 0;
+            if (estimates) {
+                for (double v: estimates[tag]) {
+                    totalEstimate += v;
+                }
+                result[tag]["averageEstimate"]=totalEstimate/values.length;
+                result[tag]["totalEstimate"]=totalEstimate;
+            }
+
         }
 
         return result;

grails-app/views/dashboard/estimates.gsp

@@ -0,0 +1,117 @@
+<%--
+
+    Copyright 2013 Netflix, Inc.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+--%>
+
+<%@ page contentType="text/html;charset=UTF-8" %>
+<head>
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+  <meta name="layout" content="main"/>
+  <title>Aws Cumulative Usage</title>
+</head>
+<body>
+<div class="" style="margin: auto; {{getBodyWidth('width: 1652px;')}} padding: 20px 30px"  ng-controller="estimateCtrl">
+  <table ng-show="!graphOnly()" style="width: auto;">
+    <tr>
+      <td>Start</td>
+      <td>Options</td>
+      <td>Account</td>
+    </tr>
+    <tr>
+      <td>
+        <input class="required" type="text" name="start" id="start" size="14"/>
+        <div style="padding-top: 10px">End</div>
+        <br><input class="required" type="text" name="end" id="end" size="14"/>
+      </td>
+      <td>
+         <div style="padding-top: 5px">Aggregate
+            <select ng-model="consolidate">
+               <option>hourly</option>
+               <option>daily</option>
+               <option>weekly</option>
+               <option>monthly</option>
+            </select>
+         </div>
+         <div style="padding-top: 5px">Cumulative
+                   <select ng-model="cumulative">
+                     <option>true</option>
+                     <option>false</option>
+                   </select>
+         </div>
+      </td>
+      <td>
+         <select ng-model="selected_accounts" ng-options="a.name for a in accounts | filter:filter_accounts" ng-change="accountsChanged()" multiple="multiple" class="metaAccounts metaSelect"></select>
+         <br><input ng-model="filter_accounts" type="text" class="metaFilter" placeholder="filter">
+      </td>
+    </tr>
+  </table>
+
+  <div class="buttons" ng-show="!graphOnly()">
+    <img src="${resource(dir: '/')}images/spinner.gif" ng-show="loading">
+    <a href="javascript:void(0)" class="monitor" style="background-image: url(${resource(dir: '/')}images/tango/16/apps/utilities-system-monitor.png)"
+       ng-click="updateUrl(); getData()" ng-show="!loading"
+       ng-disabled="selected_accounts.length == 0 || selected_regions.length == 0 || selected_products.length == 0 || showResourceGroups && selected_resourceGroups.length == 0 || selected_operations.length == 0 || selected_usageTypes.length == 0">Submit</a>
+    <!--a href="javascript:void(0)" style="background-image: url(${resource(dir: '/')}images/tango/16/actions/document-save.png)"
+       ng-click="download()" ng-show="!loading"
+       ng-disabled="selected_accounts.length == 0 || selected_regions.length == 0 || selected_products.length == 0 || showResourceGroups && selected_resourceGroups.length == 0 || selected_operations.length == 0 || selected_usageTypes.length == 0">Download</a-->
+  </div>
+
+  <table style="width: 100%; margin-top: 20px">
+    <tr>
+      <td style="width: 65%">
+        <div id="highchart_container" style="width: 100%; height: 600px;">
+        </div>
+      </td>
+    </tr>
+    <tr>
+      <td ng-show="!graphOnly()">
+         <div class="list">
+            <div>
+                  <a href="javascript:void(0)" class="legendControls" ng-click="showall()">SHOW ALL</a>
+                  <a href="javascript:void(0)" class="legendControls" ng-click="hideall()">HIDE ALL</a>
+                  <input ng-model="filter_legend" type="text" class="metaFilter" placeHolder="filter" style="float: right; margin-right: 0">
+                </div>
+                <table style="width: 100%;">
+                  <thead>
+                  <tr>
+                    <th ng-click="order(legends, 'name', false)"><div class="legendIcon" style="{{legend.iconStyle}}"></div>{{legendName}}</th>
+                    <th ng-click="order(legends, 'total', true)">Total</th>
+                    <th ng-click="order(legends, 'totalEstimate', true)">Total Estimate</th>
+                    <th ng-click="order(legends, 'average', true)">Average</th>
+                    <th ng-click="order(legends, 'averageEstimate', true)">Average Estimate</th>
+                  </tr>
+                  </thead>
+                  <tbody>
+                  <tr ng-repeat="legend in legends | filter: removeEstimatesFilter | filter:filter_legend" style="{{legend.style}}; cursor: pointer;" ng-click="clickitem(legend)" class="{{getTrClass($index)}}">
+                    <td style="word-wrap: break-word">
+                      <div class="legendIcon" style="{{legend.iconStyle}}"></div>
+                      {{legend.name}}
+                    </td>
+                    <td><span ng-show="legend_usage_cost == 'cost'">{{currencySign}} </span>{{legend.stats.total | number:2}}</td>
+                    <td><span ng-show="legend_usage_cost == 'cost'">{{currencySign}} </span>{{legend.stats.totalEstimate | number:2}}</td>
+                    <td><span ng-show="legend_usage_cost == 'cost'">{{currencySign}} </span>{{legend.stats.average | number:2}}</td>
+                    <td><span ng-show="legend_usage_cost == 'cost'">{{currencySign}} </span>{{legend.stats.averageEstimate | number:2}}</td>
+                  </tr>
+                  </tbody>
+                </table>
+            </div>
+         </td>
+    </tr>
+  </table>
+
+</div>
+</body>
+</html>

grails-app/views/layouts/main.gsp

@@ -46,6 +46,11 @@
       <a class="link_with_params" href="${resource(dir: 'dashboard', file: 'detail')}#{{getTimeParams()}}" ng-click="reload()">AWS Details</a>
       <ul>
         <li class="menuButton"><a class="link_with_params" href="${resource(dir: 'dashboard', file: 'detail')}#{{getTimeParams()}}" ng-click="reload()">General Details</a></li>
+
+        <g:if test="${ReaderConfig.getInstance().estimateReport}">
+        <li class="menuButton"><a class="link_with_params" href="${resource(dir: 'dashboard', file: 'estimates')}#{{getTimeParams()}}" ng-click="reload()">Estimate Compare</a></li>
+        </g:if>
+
         <g:if test="${ReaderConfig.getInstance().resourceService != null}">
         <li class="menuButton"><a class="link_with_params" href="${resource(dir: 'dashboard', file: 'detail')}#showResourceGroups=true&{{getTimeParams()}}" ng-click="reload()">Details With Resource Groups</a></li>
         </g:if>