001/* 002 * $URL: svn://svn.webarts.bc.ca/open/trunk/projects/WebARTS/ca/bc/webarts/tools/QTradeScraper.java $ 003 * $Author: tgutwin $ 004 * $Revision: 1255 $ 005 * $Date: 2018-03-17 20:34:04 -0700 (Sat, 17 Mar 2018) $ 006 */ 007package ca.bc.webarts.tools; 008 009import java.util.HashMap; 010import java.util.Properties; 011import javax.json.JsonObject; 012import javax.json.JsonArray; 013import javax.json.JsonNumber; 014import javax.json.JsonString; 015 016import ca.bc.webarts.widgets.ResultSetConverter; 017 018import java.text.DecimalFormat; 019 020 021 /** 022 * The CGoogle finance webpage scraper. It is a class that extends the UrlScraper class to wrap/abstract the login and 023 * credentials away and just focusses on the scraping of data from the restricted pages.<br><br> 024 * <a href="https://trading.credentialdirect.com">https://trading.credentialdirect.com</a> is a site requiring login/authentication, so this class provides autoLogin and scraping of sub-pages.<br> 025 * The site login credentials are NOT kept inside the code for this class; they are in a properties file called '<i>.credential</i>' in the '<i>user.home</i>' directory. 026 * <br>The properites that should be in this file:<ul><li>username</li><li>password</li></ul>Maybe the request properties for the website could go in here too.<br> 027 * <b>copyright (c) 2017-2018 Tom B. Gutwin</b> 028 **/ 029public class QTradeScraper extends UrlScraper 030{ 031 /** A holder for this clients System File Separator. */ 032 public final static String SYSTEM_FILE_SEPERATOR = java.io.File.separator; 033 034 /** A holder for this clients System line termination separator. */ 035 public final static String SYSTEM_LINE_SEPERATOR = 036 System.getProperty("line.separator"); 037 038 public final static String COL_DELIM = SqlQuery.DEFAULT_COLUMN_DELIMITOR; 039 public static final String QUOTE_SYMBOL_TOKEN = "^%SYMBOL%^"; 040 public static final String QUOTE_MARKET_TOKEN = "^%MARKET%^"; 041 042 /** The users home ditrectory. */ 043 public static String USERHOME = System.getProperty("user.home"); 044 045 private String qTradeFilename_ = USERHOME+SYSTEM_FILE_SEPERATOR+".qTrade"; 046 047 HashMap <String, String> reqProps = new HashMap<String, String>(); 048 049 //https://login.google.com/?.src=fpctx&.intl=ca&.lang=en-CA&authMechanism=primary&yid=&done=https%3A%2F%2Fca.google.com%2F&eid=100&as=1&login=tgutwin&crumb=9D8I.8WDkTW 050 String qLoginUrl_plong = "https://login.google.com/account/challenge/password?.src=fpctx&.intl=ca&.lang=en-CA&authMechanism=primary&yid=tgutwin&done=https%3A%2F%2Ffinance.google.com%2F&as=1&login=tgutwin&crumb=1EAn939fa%2Ft&display=login&s=QQ--&sessionIndex=QQ--&acrumb=FBRp8xFN"; 051 String qLoginUrl_pshort = "https://login.google.com/?.intl=ca&.lang=en-CA&login=tgutwin&.src=fpctx&done=https%3A%2F%2Ffinance.google.com%2F&prefill=0"; //"https://login.google.com/account/challenge/password?.src=fpctx&.intl=ca&.lang=en-CA&authMechanism=primary&yid=tgutwin&done=https%3A%2F%2Ffinance.google.com%2F&as=1&login=tgutwin"; 052 String qLoginUrl = "https://www.qtrade.ca/en/investor.html#pd"; //https://login.google.com/?authMechanism=primary&yid=&done=https%3A%2F%2Ffinance.google.com%2F&eid=100&as=1&login=tgutwin&crumb=9D8I.8WDkTW 053 String qScrapePageUrl = "https://finance.google.ca/finance?q=NASDAQ:ITRI&ei=KZITWpDrINihjAHU9JWIAg"; 054 static String qScrapeQuoteTokenizedUrl = "https://finance.google.ca/finance?q="+QUOTE_MARKET_TOKEN+":"+QUOTE_SYMBOL_TOKEN; 055 056 String qLoginFormElement = "loginForm"; /* id of the form element */ 057 String qUserLoginElement = "abcd"; 058 String qPasswordElement = "efgh"; 059 String qUsername = ""; // load from outside somewhere 060 String qPassword = ""; // load from outside somewhere 061 062 String qScrapeQuoteStart= "<div id=market-data-div"; 063 String qScrapeQuoteStart_= "<table><tr><td class=\"JgXcPd\">"; 064 String qScrapeQuoteEnd= "<script>google.finance.renderMarketData();</script>"; 065 String qScrapeQuoteEnd_= "</td></tr></table></div></div></div></g-card-section></div></div></div>"; 066 067 private JsonArray holdings = null; 068 private JsonObject dataHome = null; 069 070 protected String quoteStringCache_ = ""; 071 072 /** 073 * Default constructor for the webpage scraper. 074 * <a href="https://www.qtrade.ca/en">https://www.qtrade.ca/en</a> is a site requiring login; this class wraps the login and 075 * credentials away and just focusses on the scraping of data from the restricted pages.<br><br> 076 * 077 * The constructor ONLY gets the class fields and properties setup to go, BUT does no web page access; 078 * that is left up to the doLogin and methods. 079 **/ 080 public QTradeScraper() 081 { 082 super(); 083 084 alreadyLoggedIn_ = true; 085 loadProperties(); 086 087 reqProps.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); 088 reqProps.put("Accept-Encoding ","gzip, deflate, br"); 089 reqProps.put("Accept-Language ","en-US,en;q=0.5"); 090 reqProps.put("Connection","keep-alive"); 091 reqProps.put("Cookie","_ga=GA1.2-2.2126383985.1485192339; SC=RV=:ED=ca; _ga=GA1.3-2.2126383985.1485192339; _gid=GA1.3-2.1169777638.1511223678; NID=117=XCFBclyBN8yMrHodqAHGAnzZqTHHgdgIC7ZMLZEstpmmhvehgkI0QeHLVkvWLG0uySKvItWNuKphGmBAJe4QzCD6MBGfgLclSeW27xySZ5L_M-iSNTfeVfQHYCkl0tqP15yuhEvq7f24oyMTTyVT_QN9B3y9o8EN1Anhb9JZ; OGPC=5061451-2:5061821-3:; 1P_JAR=2017-11-21-2; OGP=-5061451:-5061821:; __gads=ID=71d1c22d471495fa:T=1508459738:S=ALNI_MbCde9o7niI4fDhlWp2dJCbeAAdNQ; S=quotestreamer=PY9m_pUIUhUQlj-rEAMVepsfJzJoAXRV"); 092 reqProps.put("Upgrade-Insecure-Requests","1"); 093 reqProps.put("Host","www.qtrade.ca"); 094 095 setLoginUrl(qLoginUrl_pshort); 096 setLoginFormID(qLoginFormElement); 097 setUsernameFormElementName(qUserLoginElement); 098 setPasswordFormElementName(qPasswordElement); 099 setUsername(qUsername); 100 setPassword(qPassword); 101 setScrapePageUrl(qScrapePageUrl); 102 setScrapeStart(qScrapeQuoteStart); 103 setScrapeEnd(qScrapeQuoteEnd); 104 setRequestProps(reqProps); 105 } 106 107 108 /** 109 * Load the private properties from an external/private file. 110 * Site login credentials are NOT kept in this class; they are in a properties file called .credential in the 'user.home' directory. 111 * <br>The properites that should be in this file:<ul><li>username</li><li>password</li></ul>Maybe the request properties for the website could go in here too. 112 * 113 * @return truee if successful, false if could not load from file 114 **/ 115 private boolean loadProperties() 116 { 117 boolean retVal = false; 118 try 119 { 120 //get user/pass from ~/.credential 121 Properties gProps = new Properties(); 122 gProps.load(new java.io.FileReader(qTradeFilename_)); 123 qUsername = gProps.getProperty("username"); 124 qPassword = gProps.getProperty("password"); 125 retVal = true; 126 } 127 catch (Exception ex) 128 { 129 // use defaults or set before use 130 System.out.println("ERROR : did not read props from :"+qTradeFilename_); 131 } 132 133 return retVal; 134 } 135 136 137 /** 138 * Not needed because this is a open webpage. 139 **/ 140 @Override 141 public boolean doLogin() 142 { 143 boolean retVal = alreadyLoggedIn_; 144 reqProps.put("Content-Type","application/x-www-form-urlencoded"); 145 146 if(!alreadyLoggedIn_) 147 if(!"".equals(getUsername()) && !"".equals(getPassword())) 148 retVal = super.doLogin(); 149 else 150 System.out.println("ERROR reading username and password"); 151 152 return retVal; 153 } 154 155 156 /** Get the latest stock QUOTE using old website. 157 * 158 * @param stockSymbol is the Symbol of the stock to lookup 159 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 160 * @return the quote OR 0.0 if not found 161 **/ 162 private double getQuoteOld(String stockSymbol, String marketSymbol) 163 { 164 165 double retVal = 0.0; 166 String currPriceStr = null; 167 String result = getQuoteString(stockSymbol, marketSymbol); 168 writeStringToFile(result,"gQuote-"+stockSymbol+".txt"); 169 //Looking for 170 // <span class="pr"> 171 // <span id="ref_630737340410842_l"> 172 // 3.72</span> 173 // </span> 174 try 175 { 176 int offset = "class=\"pr\">".length(); 177 int s= result.indexOf("class=\"pr\">"); 178 if (s==-1) 179 { 180 s = result.indexOf("class=pr>"); 181 currPriceStr = result.substring( s); 182 currPriceStr = currPriceStr.substring( currPriceStr.indexOf('>')+1); // should be from 3.72</span 183 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('<')); 184 retVal = Double.parseDouble(currPriceStr); 185 } 186 else 187 { 188 currPriceStr = result.substring( s); 189 currPriceStr = currPriceStr.substring( currPriceStr.indexOf('>', offset)+1); // should be from 3.72</span 190 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('<')); 191 retVal = Double.parseDouble(currPriceStr); 192 } 193 } 194 catch(java.lang.StringIndexOutOfBoundsException oobEx) 195 { 196 System.out.println("ERROR: get quote parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt"); 197 oobEx.printStackTrace(); 198 System.out.println(currPriceStr); 199 } 200 return retVal; 201 } 202 203 204 /** Get the latest stock QUOTE from new Website Circa 2018. 205 * 206 * @param stockSymbol is the Symbol of the stock to lookup 207 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 208 * @return the quote OR 0.0 if not found 209 **/ 210 private double getQuoteNew(String stockSymbol, String marketSymbol) 211 { 212 213 double retVal = 0.0; 214 String currPriceStr = null; 215 String result = getQuoteString(stockSymbol, marketSymbol); 216 writeStringToFile(result,"gQuote-"+stockSymbol+".txt"); 217 try 218 { 219 220 String str = "<div><span><span jsl=\"$t t-cLlm2e1WimE;$x 0;\""; 221 String str2 = "<span class=\"IsqQVc NprOob inM7_hGITiio-zJFzKq8ukm8\">"; // needed if there is a 2nd option 222 String str3 = "><span class=\""; 223 int offset = str.length(); 224 int s= result.indexOf(str); 225 if (s==-1) 226 { 227 offset = str2.length(); 228 s = result.indexOf(str2); 229 if (s!=-1) 230 { 231 currPriceStr = result.substring( s+offset); 232 int offset2 = str3.length(); 233 int s2= currPriceStr.indexOf(str3); 234 currPriceStr = currPriceStr.substring( s2+offset2); 235 int s3= currPriceStr.indexOf("\">"); 236 currPriceStr = currPriceStr.substring( s3+2); 237 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("</span>")); 238 retVal = Double.parseDouble(currPriceStr.replace(",","")); 239 } 240 } 241 else 242 { 243 currPriceStr = result.substring( s+offset); 244 int offset2 = str3.length(); 245 int s2= currPriceStr.indexOf(str3); 246 currPriceStr = currPriceStr.substring( s2+offset2); 247 int s3= currPriceStr.indexOf("\">"); 248 currPriceStr = currPriceStr.substring( s3+2); 249 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("</span>")); 250 retVal = Double.parseDouble(currPriceStr.replace(",","")); 251 } 252 } 253 catch(java.lang.StringIndexOutOfBoundsException oobEx) 254 { 255 System.out.println("ERROR: get quote parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt"); 256 oobEx.printStackTrace(); 257 System.out.println(currPriceStr); 258 } 259 return retVal; 260 } 261 262 263 /** Get the latest stock QUOTE . 264 * 265 * @param stockSymbol is the Symbol of the stock to lookup 266 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 267 * @return the quote OR 0.0 if not found 268 **/ 269 public double getQuote(String stockSymbol, String marketSymbol) 270 { 271 return getQuoteNew(stockSymbol, marketSymbol); 272 } 273 274 275 public String getTodaysTSXChartAndTableHtmlStr() 276 { 277 return getTodaysTSXChartAndTableHtmlStr(true); 278 } 279 280 281 /** Get the days TSX Index chart and data . 282 * 283 * 284 * @return html containing todays TSX indices chart 285 **/ 286 public String getTodaysTSXChartAndTableHtmlStr(boolean includeTable) 287 { 288 String retVal = ""; 289 boolean success = false; 290 boolean doLogin = false; 291 setScrapePageUrl("https://finance.google.ca/finance"); 292 setScrapeStart("<div class=id-summary-chart>"); 293 setScrapeEnd("</div></div><div class=\"sfe-section clf\">"); 294 //setScrapeStart("BlahBlahBlah"); 295 //setScrapeEnd("BlahBlahBlah"); 296 297 HashMap <String, String> reqProps = null; 298 299 setDebugOut(); 300 301 if(doLogin) 302 { 303 if(!"".equals(getUsername()) && !"".equals(getPassword())) 304 { 305 success = doLogin(); 306 //System.out.println("\nLogin Response:\n"+postPageResponse_); 307 } 308 else 309 System.out.println("ERROR reading username and password"); 310 } 311 else 312 success = true; 313 314 if(success) 315 { 316 String result = doScrape(false); 317 retVal = result; 318 319 //System.out.println("Scraped Chart:\n"+retVal); 320 } 321 322 /* 323 //Looking for 324 // <span class="pr"> 325 // <span id="ref_630737340410842_l"> 326 // 3.72</span> 327 // </span> 328 try 329 { 330 int offset = "Vol / Avg.</td>".length(); 331 int s= result.indexOf("Vol / Avg.</td>"); 332 if (s==-1) 333 { 334 s = result.indexOf("class=pr>"); 335 if (s!=-1) 336 { 337 currPriceStr = result.substring( s); 338 currPriceStr = currPriceStr.substring( currPriceStr.indexOf('>')+1); // should be from 3.72</span 339 if(currPriceStr.indexOf('M')!=-1 && currPriceStr.indexOf('M')<8) 340 { 341 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('M')); 342 retVal = Double.parseDouble(currPriceStr.replace(",",""))*1000000.0; 343 } 344 else 345 { 346 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('/')); 347 retVal = Double.parseDouble(currPriceStr.replace(",","")); 348 } 349 } 350 } 351 else 352 { 353 currPriceStr = result.substring( s); 354 //System.out.println("\nDEBUG:"+currPriceStr); 355 currPriceStr = currPriceStr.substring( currPriceStr.indexOf('>', offset)+1); // should be from 3.72</span 356 //System.out.println("\nDEBUG:"+currPriceStr); 357 if(currPriceStr.indexOf('M')!=-1 && currPriceStr.indexOf('M')<8) 358 { 359 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('M')); 360 retVal = Double.parseDouble(currPriceStr.replace(",",""))*1000000.0; 361 } 362 else 363 { 364 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('/')); 365 retVal = Double.parseDouble(currPriceStr.replace(",","")); 366 } 367 } 368 } 369 catch(java.lang.StringIndexOutOfBoundsException oobEx) 370 { 371 System.out.println("ERROR: get volume parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt"); 372 oobEx.printStackTrace(); 373 System.out.println(currPriceStr); 374 } 375 */ 376 return retVal; 377 } 378 379 380 /** Get the latest stock stat for the passed in valueName from the NEW GFinance pages Circa 2018. 381 * 382 * @param stockSymbol is the Symbol of the stock to lookup 383 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 384 * @param valueName is the stat to scrape - Open, Close, High, Low, 52-wk high, 52-wk low, Div yield 385 * @return the price OR -1 if not found; 386 **/ 387 private double scrapeValueNew(String stockSymbol, String marketSymbol, String valueName) 388 { 389 double retVal = -1.0; 390 String currPriceStr = null; 391 if ("Close".equalsIgnoreCase(valueName)) valueName = "Prev close"; 392 if ("Prev close".equalsIgnoreCase(valueName)) valueName = "Prev close"; 393 if ("Div".equalsIgnoreCase(valueName)) valueName = "Div yield"; 394 if ("Dividend".equalsIgnoreCase(valueName)) valueName = "Div yield"; 395 if ("Div yield".equalsIgnoreCase(valueName)) valueName = "Div yield"; 396 if(!"".equalsIgnoreCase(marketSymbol)) 397 { 398 retVal = 0.0; 399 String result = getQuoteString(stockSymbol, marketSymbol); 400 writeStringToFile(result,"gQuote-"+stockSymbol+".txt"); 401 //Looking for 402 // <div class="ZSM8k"><table><tr> 403 // <td class="JgXcPd">Open</td><td class="iyjjgb">21.50</td></tr> 404 // <tr><td class="JgXcPd">High</td><td class="iyjjgb">21.93</td></tr> 405 // <tr><td class="JgXcPd">Low</td><td class="iyjjgb">20.65</td></tr> 406 // <tr><td class="JgXcPd">Mkt cap</td><td class="iyjjgb">567.04M</td></tr> 407 // <tr><td class="JgXcPd">P/E ratio</td><td class="iyjjgb">10.42</td></tr> 408 // </table></div> 409 // <div class="ZSM8k"><table><tr> 410 // <td class="JgXcPd">Div yield</td><td class="iyjjgb">1.94%</td></tr> 411 // <tr><td class="JgXcPd">Prev close</td><td class="iyjjgb">20.87</td></tr> 412 // <tr><td class="JgXcPd">52-wk high</td><td class="iyjjgb">26.11</td></tr> 413 // <tr><td class="JgXcPd">52-wk low</td><td class="iyjjgb">17.46</td></tr> 414 // </table></div> 415 try 416 { 417 String str = "<td class=\"JgXcPd\">"+valueName+"</td><td class=\"iyjjgb\">"; 418 String str2 = "<td class=JgXcPd>"+valueName+"</td><td class=iyjjgb>"; 419 int offset = str.length(); 420 int s= result.indexOf(str); 421 if (s==-1) 422 { 423 offset = str2.length(); 424 s = result.indexOf(str2); 425 if (s!=-1) 426 { 427 currPriceStr = result.substring( s+offset); 428 if(valueName == "Div yield") 429 { 430 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("%</")); 431 if(currPriceStr.startsWith("-")) currPriceStr = "0.0"; 432 } 433 else 434 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("</")); 435 retVal = Double.parseDouble(currPriceStr.replace(",","")); 436 } 437 } 438 else 439 { 440 currPriceStr = result.substring( s+offset); 441 if(valueName == "Div yield") 442 { 443 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("%</")); 444 if(currPriceStr.startsWith("-")) currPriceStr = "0.0"; 445 } 446 else 447 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("</")); 448 retVal = Double.parseDouble(currPriceStr.replace(",","")); 449 } 450 } 451 catch(java.lang.StringIndexOutOfBoundsException oobEx) 452 { 453 System.out.println("ERROR: scraping "+valueName+". See result html in file "+ "gQuote-"+stockSymbol+".txt"); 454 oobEx.printStackTrace(); 455 System.out.println(currPriceStr); 456 } 457 catch(java.lang.NumberFormatException mfEx) 458 { 459 System.out.println("ERROR: number Format error scraping "+valueName+". See result html in file "+ "gQuote-"+stockSymbol+".txt"); 460 mfEx.printStackTrace(); 461 System.out.println(currPriceStr); 462 } 463 } 464 return retVal; 465 } 466 467 468 /** Get the latest stock Open price from the OLD GFinance Pages . 469 * 470 * @param stockSymbol is the Symbol of the stock to lookup 471 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 472 * @return the price OR -1 if not found; 473 **/ 474 private double getOpenOld(String stockSymbol, String marketSymbol) 475 { 476 477 double retVal = -1.0; 478 String currPriceStr = null; 479 if(!"".equalsIgnoreCase(marketSymbol)) 480 { 481 retVal = 0.0; 482 String result = getQuoteString(stockSymbol, marketSymbol); 483 writeStringToFile(result,"gQuote-"+stockSymbol+".txt"); 484 //Looking for 485 // <span class="pr"> 486 // <span id="ref_630737340410842_l"> 487 // 3.72</span> 488 // </span> 489 try 490 { 491 int offset = "Open</td><td class=\"val\">".length(); 492 int s= result.indexOf("Open</td><td class=\"val\">"); 493 if (s==-1) 494 { 495 offset = "Open</td><td class=val>".length(); 496 s = result.indexOf("Open</td><td class=val>"); 497 if (s!=-1) 498 { 499 currPriceStr = result.substring( s+offset); 500 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("</")); 501 retVal = Double.parseDouble(currPriceStr.replace(",","")); 502 } 503 } 504 else 505 { 506 currPriceStr = result.substring( s+offset); 507 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("</")); 508 retVal = Double.parseDouble(currPriceStr.replace(",","")); 509 } 510 } 511 catch(java.lang.StringIndexOutOfBoundsException oobEx) 512 { 513 System.out.println("ERROR: get open parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt"); 514 oobEx.printStackTrace(); 515 System.out.println(currPriceStr); 516 } 517 } 518 return retVal; 519 } 520 521 522 /** Get the latest stock Open price from the NEW GFinance pages Circa 2018. 523 * 524 * @param stockSymbol is the Symbol of the stock to lookup 525 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 526 * @return the price OR -1 if not found; 527 **/ 528 private double getOpenNew(String stockSymbol, String marketSymbol) 529 { 530 return scrapeValueNew(stockSymbol, marketSymbol, "Open"); 531 } 532 533 534 /** Get the latest stock Open price. 535 * 536 * @param stockSymbol is the Symbol of the stock to lookup 537 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 538 * @return the price OR -1 if not found; 539 **/ 540 public double getOpen(String stockSymbol, String marketSymbol) 541 { 542 return getOpenNew(stockSymbol, marketSymbol); 543 } 544 545 546 /** Get the latest stock Days High price from the Old GFinance pages before 2018. 547 * 548 * @param stockSymbol is the Symbol of the stock to lookup 549 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 550 * @return the price OR -1 if not found; 551 **/ 552 private double getDaysHighOld(String stockSymbol, String marketSymbol) 553 { 554 555 double retVal = -1.0; 556 String currLowStr = null; 557 String currHighStr = null; 558 if(!"".equalsIgnoreCase(marketSymbol)) 559 { 560 retVal = 0.0; 561 String result = getQuoteString(stockSymbol, marketSymbol); 562 writeStringToFile(result,"gQuote-"+stockSymbol+".txt"); 563 //Looking for 564 // Range</td><td class="val"> 565 try 566 { 567 int offset = "Range</td><td class=\"val\">".length(); 568 int s= result.indexOf("Range</td><td class=\"val\">"); 569 if (s==-1) 570 { 571 offset = "Range</td><td class=val>".length(); 572 s = result.indexOf("Range</td><td class=val>"); 573 if (s!=-1) 574 { 575 currLowStr = result.substring( s+offset); 576 currHighStr = currLowStr.substring( currLowStr.indexOf(" - ")+3); 577 currHighStr = currHighStr.substring( 0, currHighStr.indexOf("</td>")); 578 currLowStr = currLowStr.substring( 0, currLowStr.indexOf(" - ")); 579 retVal = Double.parseDouble(currHighStr.replace(",","")); 580 } 581 } 582 else 583 { 584 currLowStr = result.substring( s+offset); 585 currHighStr = currLowStr.substring( currLowStr.indexOf(" - ")+3); 586 currHighStr = currHighStr.substring( 0, currHighStr.indexOf("</td>")); 587 currLowStr = currLowStr.substring( 0, currLowStr.indexOf(" - ")); 588 retVal = Double.parseDouble(currHighStr.replace(",","")); 589 } 590 } 591 catch(java.lang.StringIndexOutOfBoundsException oobEx) 592 { 593 System.out.println("ERROR: get days high parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt"); 594 oobEx.printStackTrace(); 595 System.out.println(currHighStr); 596 } 597 } 598 return retVal; 599 } 600 601 602 /** Get the latest stock Days High price from the NEW GFinance pages Circa 2018. 603 * 604 * @param stockSymbol is the Symbol of the stock to lookup 605 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 606 * @return the price OR -1 if not found; 607 **/ 608 private double getDaysHighNew(String stockSymbol, String marketSymbol) 609 { 610 return scrapeValueNew(stockSymbol, marketSymbol, "High"); 611 } 612 613 614 /** Get the latest stock Days High price . 615 * 616 * @param stockSymbol is the Symbol of the stock to lookup 617 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 618 * @return the price OR -1 if not found; 619 **/ 620 public double getDaysHigh(String stockSymbol, String marketSymbol) 621 {return getDaysHighNew( stockSymbol, marketSymbol);} 622 623 624 /** Get the latest stock Days Low price . 625 * 626 * @param stockSymbol is the Symbol of the stock to lookup 627 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 628 * @return the price OR -1 if not found; 629 **/ 630 private double getDaysLowOld(String stockSymbol, String marketSymbol) 631 { 632 633 double retVal = -1.0; 634 String currLowStr = null; 635 String currHighStr = null; 636 if(!"".equalsIgnoreCase(marketSymbol)) 637 { 638 retVal = 0.0; 639 String result = getQuoteString(stockSymbol, marketSymbol); 640 writeStringToFile(result,"gQuote-"+stockSymbol+".txt"); 641 //Looking for 642 // Range</td><td class="val"> 643 try 644 { 645 int offset = "Range</td><td class=\"val\">".length(); 646 int s= result.indexOf("Range</td><td class=\"val\">"); 647 if (s==-1) 648 { 649 offset = "Range</td><td class=val>".length(); 650 s = result.indexOf("Range</td><td class=val>"); 651 if (s!=-1) 652 { 653 currLowStr = result.substring( s+offset); 654 currHighStr = currLowStr.substring( currLowStr.indexOf(" - ")+3); 655 currHighStr = currHighStr.substring( 0, currHighStr.indexOf("</td>")); 656 currLowStr = currLowStr.substring( 0, currLowStr.indexOf(" - ")); 657 retVal = Double.parseDouble(currLowStr.replace(",","")); 658 } 659 } 660 else 661 { 662 currLowStr = result.substring( s+offset); 663 currHighStr = currLowStr.substring( currLowStr.indexOf(" - ")+3); 664 currHighStr = currHighStr.substring( 0, currHighStr.indexOf("</td>")); 665 currLowStr = currLowStr.substring( 0, currLowStr.indexOf(" - ")); 666 retVal = Double.parseDouble(currLowStr.replace(",","")); 667 } 668 } 669 catch(java.lang.StringIndexOutOfBoundsException oobEx) 670 { 671 System.out.println("ERROR: get days high parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt"); 672 oobEx.printStackTrace(); 673 System.out.println(currLowStr); 674 } 675 } 676 return retVal; 677 } 678 679 680 /** Get the latest stock Days Low price from the NEW GFinance pages Circa 2018. 681 * 682 * @param stockSymbol is the Symbol of the stock to lookup 683 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 684 * @return the price OR -1 if not found; 685 **/ 686 private double getDaysLowNew(String stockSymbol, String marketSymbol) 687 { 688 return scrapeValueNew(stockSymbol, marketSymbol, "Low"); 689 } 690 691 692 /** Get the latest stock Days Low price . 693 * 694 * @param stockSymbol is the Symbol of the stock to lookup 695 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 696 * @return the price OR -1 if not found; 697 **/ 698 public double getDaysLow(String stockSymbol, String marketSymbol) 699 {return getDaysLowNew( stockSymbol, marketSymbol);} 700 701 702 /** Get the latest stock Dividend Yield % . 703 * Latest dividend/dividend yield<br><br> 704 * Latest dividend is dividend per share paid to shareholders in the most recent quarter.<br> 705 * Dividend yield is the value of the latest dividend, multiplied by the number of times dividends are typically paid per year, divided by the stock price. 706 * 707 * @param stockSymbol is the Symbol of the stock to lookup 708 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 709 * @return the % OR -1 if not found; 710 **/ 711 private double getLatestDividendOld(String stockSymbol, String marketSymbol) 712 { 713 714 double retVal = -1.0; 715 String findStr = "Div/yield</td><td class=\"val\">"; 716 String findStr2 = "Div/yield</td><td class=val>"; 717 String dividendStr = null; 718 String divYieldStr = null; 719 if(!"".equalsIgnoreCase(marketSymbol)) 720 { 721 retVal = 0.0; 722 String result = getQuoteString(stockSymbol, marketSymbol); 723 writeStringToFile(result,"gQuote-"+stockSymbol+".txt"); 724 //Looking for 725 // Range</td><td class="val"> 726 try 727 { 728 int offset = findStr.length(); 729 int s= result.indexOf(findStr); 730 if (s==-1) 731 { 732 offset = findStr2.length(); 733 s = result.indexOf(findStr2); 734 if (s!=-1) 735 { 736 dividendStr = result.substring( s+offset); 737 if(!dividendStr.startsWith("&")) 738 { 739 divYieldStr = dividendStr.substring( dividendStr.indexOf("/")+1); 740 divYieldStr = divYieldStr.substring( 0, divYieldStr.indexOf("</td>")); 741 dividendStr = dividendStr.substring( 0, dividendStr.indexOf("/")); 742 retVal = Double.parseDouble(dividendStr.replace(",","")); 743 } 744 } 745 } 746 else 747 { 748 dividendStr = result.substring( s+offset); 749 if(!dividendStr.startsWith("&")) 750 { 751 divYieldStr = dividendStr.substring( dividendStr.indexOf("/")+1); 752 divYieldStr = divYieldStr.substring( 0, divYieldStr.indexOf("</td>")); 753 dividendStr = dividendStr.substring( 0, dividendStr.indexOf("/")); 754 retVal = Double.parseDouble(dividendStr.replace(",","")); 755 } 756 } 757 } 758 catch(java.lang.StringIndexOutOfBoundsException oobEx) 759 { 760 System.out.println("ERROR: get dividend parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt"); 761 oobEx.printStackTrace(); 762 System.out.println(dividendStr); 763 } 764 } 765 return retVal; 766 } 767 768 769 /** Get the latest stock Div Yield % from the NEW GFinance pages Circa 2018. 770 * 771 * @param stockSymbol is the Symbol of the stock to lookup 772 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 773 * @return the div % OR -1 if not found; 774 **/ 775 private double getLatestDividendNew(String stockSymbol, String marketSymbol) 776 { 777 return scrapeValueNew(stockSymbol, marketSymbol, "Div yield"); 778 } 779 780 781 /** Get the latest stock Div Yield %. 782 * 783 * @param stockSymbol is the Symbol of the stock to lookup 784 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 785 * @return the div % OR -1 if not found; 786 **/ 787 public double getLatestDividend(String stockSymbol, String marketSymbol) 788 {return getLatestDividendNew( stockSymbol, marketSymbol);} 789 790 791 /** Get the latest stock Volume . 792 * 793 * @param stockSymbol is the Symbol of the stock to lookup 794 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 795 * @return the most recent daily volume OR -1 if not found; 796 **/ 797 private double getVolumeOld(String stockSymbol, String marketSymbol) 798 { 799 800 double retVal = -1.0; 801 String currPriceStr = null; 802 if(!"MUTF_CA".equalsIgnoreCase(marketSymbol)) 803 { 804 retVal = 0.0; 805 String result = getQuoteString(stockSymbol, marketSymbol); 806 writeStringToFile(result,"gQuote-"+stockSymbol+".txt"); 807 //Looking for 808 // <span class="pr"> 809 // <span id="ref_630737340410842_l"> 810 // 3.72</span> 811 // </span> 812 try 813 { 814 int offset = "Vol / Avg.</td>".length(); 815 int s= result.indexOf("Vol / Avg.</td>"); 816 if (s==-1) 817 { 818 s = result.indexOf("class=pr>"); 819 if (s!=-1) 820 { 821 currPriceStr = result.substring( s); 822 currPriceStr = currPriceStr.substring( currPriceStr.indexOf('>')+1); // should be from 3.72</span 823 if(currPriceStr.indexOf('M')!=-1 && currPriceStr.indexOf('M')<8) 824 { 825 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('M')); 826 retVal = Double.parseDouble(currPriceStr.replace(",",""))*1000000.0; 827 } 828 else 829 { 830 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('/')); 831 retVal = Double.parseDouble(currPriceStr.replace(",","")); 832 } 833 } 834 } 835 else 836 { 837 currPriceStr = result.substring( s); 838 //System.out.println("\nDEBUG:"+currPriceStr); 839 currPriceStr = currPriceStr.substring( currPriceStr.indexOf('>', offset)+1); // should be from 3.72</span 840 //System.out.println("\nDEBUG:"+currPriceStr); 841 if(currPriceStr.indexOf('M')!=-1 && currPriceStr.indexOf('M')<8) 842 { 843 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('M')); 844 retVal = Double.parseDouble(currPriceStr.replace(",",""))*1000000.0; 845 } 846 else 847 { 848 currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('/')); 849 retVal = Double.parseDouble(currPriceStr.replace(",","")); 850 } 851 } 852 } 853 catch(java.lang.StringIndexOutOfBoundsException oobEx) 854 { 855 System.out.println("ERROR: get volume parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt"); 856 oobEx.printStackTrace(); 857 System.out.println(currPriceStr); 858 } 859 } 860 return retVal; 861 } 862 863 864 /** Get the latest stock Volume from the NEW GFinance pages Circa 2018 - IF IT EXISTED. 865 * 866 * @param stockSymbol is the Symbol of the stock to lookup 867 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 868 * @return the Volume from most resent day -1 if not found; 869 **/ 870 private double getVolumeNew(String stockSymbol, String marketSymbol) 871 { 872 return -1.0; // does not exist in new Google... scrapeValueNew(stockSymbol, marketSymbol, "Div yield"); 873 } 874 875 876 /** Get the latest stock Volume. 877 * 878 * @param stockSymbol is the Symbol of the stock to lookup 879 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 880 * @return the div % OR -1 if not found; 881 **/ 882 public double getVolume(String stockSymbol, String marketSymbol) 883 {return getVolumeNew( stockSymbol, marketSymbol);} 884 885 886 /** Get the latest stock Dividend Yield. 887 * Latest dividend/dividend yield<br><br> 888 * Latest dividend is dividend per share paid to shareholders in the most recent quarter.<br> 889 * Dividend yield is the value of the latest dividend, multiplied by the number of times dividends are typically paid per year, divided by the stock price. 890 * 891 * @param stockSymbol is the Symbol of the stock to lookup 892 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 893 * @return the price OR -1 if not found; 894 **/ 895 public double getDividendYieldOld(String stockSymbol, String marketSymbol) 896 { 897 898 double retVal = -1.0; 899 String findStr = "Div/yield</td><td class=\"val\">"; 900 String findStr2 = "Div/yield</td><td class=val>"; 901 String dividendStr = null; 902 String divYieldStr = null; 903 if(!"".equalsIgnoreCase(marketSymbol)) 904 { 905 retVal = 0.0; 906 String result = getQuoteString(stockSymbol, marketSymbol); 907 writeStringToFile(result,"gQuote-"+stockSymbol+".txt"); 908 //Looking for 909 // Range</td><td class="val"> 910 try 911 { 912 int offset = findStr.length(); 913 int s= result.indexOf(findStr); 914 if (s==-1) 915 { 916 offset = findStr2.length(); 917 s = result.indexOf(findStr2); 918 if (s!=-1) 919 { 920 dividendStr = result.substring( s+offset); 921 if(!dividendStr.startsWith("&")) 922 { 923 divYieldStr = dividendStr.substring( dividendStr.indexOf("/")+1); 924 divYieldStr = divYieldStr.substring( 0, divYieldStr.indexOf("</td>")); 925 dividendStr = dividendStr.substring( 0, dividendStr.indexOf("/")); 926 retVal = Double.parseDouble(divYieldStr.replace(",","")); 927 } 928 } 929 } 930 else 931 { 932 dividendStr = result.substring( s+offset); 933 if(!dividendStr.startsWith("&")) 934 { 935 divYieldStr = dividendStr.substring( dividendStr.indexOf("/")+1); 936 divYieldStr = divYieldStr.substring( 0, divYieldStr.indexOf("</td>")); 937 dividendStr = dividendStr.substring( 0, dividendStr.indexOf("/")); 938 retVal = Double.parseDouble(divYieldStr.replace(",","")); 939 } 940 } 941 } 942 catch(java.lang.StringIndexOutOfBoundsException oobEx) 943 { 944 System.out.println("ERROR: get dividend Yield parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt"); 945 oobEx.printStackTrace(); 946 System.out.println(divYieldStr); 947 } 948 } 949 return retVal; 950 } 951 952 953 /** 954 * Returns the URL string to the Google page for the specified stock. 955 **/ 956 public static String getStockPageUrlStr(String stockSymbol, String marketSymbol) 957 { 958 String retVal = ""; 959 retVal = qScrapeQuoteTokenizedUrl.replace(QUOTE_SYMBOL_TOKEN, stockSymbol); 960 retVal = retVal.replace(QUOTE_MARKET_TOKEN, googleMarketSymbol(marketSymbol)); 961 962 return retVal; 963 } 964 965 966 /** Requests. scrapes & creates the stock QUOTE data string for the given stock symbol for today. 967 * this data gets used as the data load into the InvestmentTrackerQuery database.<br> 968 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 969 * open, daysLow, daysHigh, previousClose, daysVolume, dividendPerShare, dividendPayDate<br> 970 * 90.01, 89.13, 90.01, 90.34, 386147, 4.18, "10/27/2017" 971 * 972 * @param stockSymbol is the Symbol of the stock to lookup 973 * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl 974 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited open, daysLow, daysHigh, previousClose, daysVolume, dividendPerShare, dividendPayDate OR '' if not found 975 **/ 976 private String getQuoteString(String stockSymbol, String marketSymbol) { return getQuoteString(stockSymbol, marketSymbol, true);} 977 private String getQuoteString(String stockSymbol, String marketSymbol, boolean useCache) 978 { 979 String retVal = ""; 980 boolean success = false; 981 boolean useDataTool = false; 982 //NASDAQ:FCEL CVE:HIVE MUTF_CA:AGF201 TSE:LUN 983 984 if(useCache && stockSymbolCache_ == stockSymbol && stockSymbolCache_ == stockSymbol) 985 { 986 retVal = quoteStringCache_; 987 } 988 else 989 { 990 String tokenReplacedScrapeUrl = qScrapeQuoteTokenizedUrl.replace(QUOTE_SYMBOL_TOKEN,stockSymbol); 991 tokenReplacedScrapeUrl = tokenReplacedScrapeUrl.replace(QUOTE_MARKET_TOKEN,marketSymbol); 992 setScrapePageUrl(tokenReplacedScrapeUrl); 993 994 HashMap <String, String> reqProps = null; 995 996 if(holdings==null && !marketSymbol.equals("MUTF_CA")) // Google is banning my MUTF_CA queries 997 { 998 if(!"".equals(getUsername()) && !"".equals(getPassword())) 999 { 1000 success = doLogin(); 1001 //System.out.println("\nLogin Response:\n"+postPageResponse_); 1002 } 1003 else 1004 System.out.println("ERROR reading username and password"); 1005 1006 if(success) 1007 { 1008 String result = doScrape(reqProps, true); // without any further trimming, only the start/end trimming 1009 retVal = result; 1010 } 1011 stockSymbolCache_ = stockSymbol; 1012 marketSymbolCache_ = marketSymbol; 1013 quoteStringCache_ = retVal; 1014 } 1015 } 1016 return retVal; 1017 } 1018 1019 1020 1021 /** Create the STOCK_DAILY data string for the given stock symbol for today. 1022 * this data gets used as the data load into the InvestmentTrackerQuery database.<br> 1023 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 1024 * open, daysLow, daysHigh, close, previousClose, daysVolume, date, dividendPerShare, dividendPayDate<br> 1025 * 90.01, 89.13, 90.01, 90.00, 90.34, 386147, "11/08/2017", 4.18, "10/27/2017" 1026 * 1027 * @param stockSymbol is the Symbol of the stock to lookup * 1028 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited open, daysLow, daysHigh, previousClose, daysVolume, dividendPerShare, dividendPayDate OR '' if not found 1029 **/ 1030 public String getStockDailyString(String stockSymbol) 1031 { 1032 String retVal = ""; 1033 boolean success = false; 1034 JsonObject dataHome = null; 1035 if(holdings==null) 1036 { 1037 if(!"".equals(getUsername()) && !"".equals(getPassword())) 1038 success = doLogin(); 1039 else 1040 System.out.println("ERROR reading username and password"); 1041 1042 if(success) 1043 { 1044 // scape the summary first to get a list of holdings 1045 String result = doScrape(); 1046 JsonObject jsO = toJsonObject(result); 1047 JsonArray jsA = jsO.getJsonArray("Data"); 1048 dataHome = jsA.getJsonObject(0); 1049 holdings = dataHome.getJsonArray("Holdings"); 1050 } 1051 } 1052 else 1053 System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]"); 1054 1055 if(dataHome!=null && holdings!=null) 1056 { 1057 retVal+="\"Open\""; 1058 retVal+=COL_DELIM; 1059 retVal+="\"daysLow\""; 1060 retVal+=COL_DELIM; 1061 retVal+="\"daysHigh\""; 1062 retVal+=COL_DELIM; 1063 retVal+="\"close\""; 1064 retVal+=COL_DELIM; 1065 retVal+="\"previousClose\""; 1066 retVal+=COL_DELIM; 1067 retVal+="\"date\""; 1068 retVal+=COL_DELIM; 1069 retVal+="\"dividendPerShare\""; 1070 retVal+=COL_DELIM; 1071 retVal+="\"dividendPayDate\""; 1072 retVal+="\n"; 1073 1074 setScrapeStart(qScrapeQuoteStart); 1075 setScrapeEnd(qScrapeQuoteEnd); 1076 String currSymbol = ""; 1077 String currPrefix = ""; 1078 JsonObject stock =null; 1079 1080 for(int i=0; i<holdings.size();i++) 1081 { 1082 stock = holdings.getJsonObject(i); 1083 if(stock!=null) 1084 { 1085 currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1); 1086 currPrefix = stock.getJsonString("CountryPrefix").toString().substring(1,stock.getJsonString("CountryPrefix").toString().length()-1); 1087 1088 if(("\""+stockSymbol+"\"").equalsIgnoreCase(currSymbol.toString())) 1089 { 1090 1091 //get the daily quote for each individual holding 1092 setScrapePageUrl(qScrapeQuoteTokenizedUrl.replace(QUOTE_SYMBOL_TOKEN,currPrefix+currSymbol)); 1093 String result = doScrape(); 1094 System.out.println(" DEBUG Stock Quote Result \n----------------------------\n"); 1095 System.out.println(result); 1096 1097 /* 1098 JsonObject jsO = toJsonObject(result); 1099 JsonString currSymbol = stock.getJsonString("Symbol"); 1100 if(("\""+stockSymbol+"\"").equalsIgnoreCase(currSymbol.toString())) 1101 { 1102 JsonNumber open, daysLow, daysHigh, previousClose, daysVolume, dividendPerShare; 1103 String dividendPayDate; 1104 //open=stock.getJsonString("Price"); 1105 retVal+="\""+dataHome.getString("FriendlyName")+"\""; 1106 retVal+=COL_DELIM; 1107 retVal+=""+dataHome.getJsonString("Currency"); 1108 retVal+=COL_DELIM; 1109 1110 } 1111 */ 1112 i=holdings.size(); 1113 } 1114 } 1115 } 1116 } 1117 1118 return retVal; 1119 } 1120 1121 1122 public static String googleMarketSymbol(String mkt) 1123 { 1124 String retVal = mkt; 1125 1126 if("TSXV".equalsIgnoreCase(mkt)) retVal = "CVE"; 1127 else if("TSX".equalsIgnoreCase(mkt)) retVal = "TSE"; 1128 else if("MUT".equalsIgnoreCase(mkt)) retVal = "MUTF_CA"; 1129 else if("MUTF".equalsIgnoreCase(mkt)) retVal = "MUTF_CA"; 1130 1131 return retVal; 1132 } 1133 1134 1135 /** Create the Portfolio Summary data string for the CR account for today. 1136 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 1137 * FriendlyName, Currency, BookValue, TradeCash, MarketValue, EquityValue, UnrealizedGainLoss, UnrealizedGainLossPercent, numHoldings, date<br> 1138 * "#2V9669V6 - TFSA", 50.5, 10.1, 66.4, 15.9, 8, "10/27/2017" 1139 * 1140 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate 1141 **/ 1142 public String getPortfolioSummaryString() 1143 { 1144 String retVal = ""; 1145 boolean success = false; 1146 JsonObject dataHome = null; 1147 if(holdings==null) 1148 { 1149 if(!"".equals(getUsername()) && !"".equals(getPassword())) 1150 success = doLogin(); 1151 else 1152 System.out.println("ERROR reading username and password"); 1153 1154 if(success) 1155 { 1156 String result = doScrape(); 1157 JsonObject jsO = toJsonObject(result); 1158 JsonArray jsA = jsO.getJsonArray("Data"); 1159 dataHome = jsA.getJsonObject(0); 1160 //holdings = dataHome.getJsonArray("Holdings"); 1161 } 1162 } 1163 else 1164 System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]"); 1165 1166 if(dataHome!=null) // && holdings!=null) 1167 { 1168 retVal+="\"friendlyName\""; 1169 retVal+=COL_DELIM; 1170 retVal+="\"Currency\""; 1171 retVal+=COL_DELIM; 1172 retVal+="\"BookValue\""; 1173 retVal+=COL_DELIM; 1174 retVal+="\"TradeCash\""; 1175 retVal+=COL_DELIM; 1176 retVal+="\"MarketValue\""; 1177 retVal+=COL_DELIM; 1178 retVal+="\"EquityValue\""; 1179 retVal+=COL_DELIM; 1180 retVal+="\"UnrealizedGainLoss\""; 1181 retVal+=COL_DELIM; 1182 retVal+="\"UnrealizedGainLossPercent\""; 1183 retVal+=COL_DELIM; 1184 retVal+="\"NumberOfHoldings\""; 1185 retVal+=COL_DELIM; 1186 retVal+="\"date\""; 1187 retVal+="\n"; 1188 1189 retVal+="\""+dataHome.getString("FriendlyName")+"\""; 1190 retVal+=COL_DELIM; 1191 retVal+=""+dataHome.getJsonString("Currency"); 1192 retVal+=COL_DELIM; 1193 retVal+=""+dataHome.getJsonNumber("BookValue"); 1194 retVal+=COL_DELIM; 1195 retVal+=""+dataHome.getJsonNumber("TradeCash"); 1196 retVal+=COL_DELIM; 1197 retVal+=""+dataHome.getJsonNumber("MarketValue"); 1198 retVal+=COL_DELIM; 1199 retVal+=""+dataHome.getJsonNumber("EquityValue"); 1200 retVal+=COL_DELIM; 1201 retVal+=""+dataHome.getJsonNumber("UnrealizedGainLoss"); 1202 retVal+=COL_DELIM; 1203 retVal+=""+dataHome.getJsonNumber("UnrealizedGainLossPercent"); 1204 retVal+=COL_DELIM; 1205 retVal+=""+(holdings!=null?holdings.size():"0"); 1206 retVal+=COL_DELIM; 1207 retVal+="\""+dateStr_+"\""; 1208 1209 /* 1210 for(int i=0; i<holdings.size();i++) 1211 { 1212 JsonObject stock = holdings.getJsonObject(i); 1213 if(stock!=null) 1214 { 1215 String currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1); 1216 System.out.println(currSymbol+" : "+stock.getJsonNumber("Quantity").toString()+" @ "+ 1217 stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1)+ 1218 " = "+stock.getJsonNumber("MarketValue").toString()); 1219 } 1220 } 1221 */ 1222 } 1223 1224 return retVal; 1225 } 1226 1227 1228 1229 /** Create the Portfolio data string for the CR account for today. 1230 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 1231 * stockName, symbol, CRQuerySymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate<br> 1232 * Hive Tech, HIVE, HIVE, 500, 43.4, CA, 1502.40, 2900.84, 1397.44, 83.4, "10/27/2017" 1233 * 1234 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate 1235 **/ 1236 public String getPortfolioString(){return getPortfolioString(true);} 1237 1238 1239 1240 /** Create the Portfolio data string for the CR account for today. 1241 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 1242 * stockName, symbol, CRExchangeSymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate<br> 1243 * Hive Tech, HIVE, V, 500, 43.4, CA, 1502.40, 2900.84, 1397.44, 83.4, 10-27-2017 1244 * 1245 * @param resultSetOnly true to return only the default delimited resultSet OR false to send a longer more readable string 1246 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate 1247 **/ 1248 public String getPortfolioString(boolean resultSetOnly) 1249 { 1250 String retVal = "stockName"+COL_DELIM+"symbol"+COL_DELIM+"CRQuerySymbol"+COL_DELIM+"numShares"+COL_DELIM+"currentPrice"+COL_DELIM+ 1251 "currency"+COL_DELIM+"bookValue"+COL_DELIM+"marketValue"+COL_DELIM+"gain"+COL_DELIM+"gainPercent"+COL_DELIM+"currentDate\n"; 1252 if(!resultSetOnly) retVal = "\n\n---------------------------------\n Portfolio Summary\n---------------------------------\n"; 1253 1254 DecimalFormat dfp = new DecimalFormat( "##0" ); 1255 DecimalFormat dfe = new DecimalFormat( "##0.000" ); 1256 boolean success = false; 1257 1258 if(holdings==null) 1259 { 1260 if(!"".equals(getUsername()) && !"".equals(getPassword())) 1261 success = doLogin(); 1262 else 1263 System.out.println("ERROR reading username and password"); 1264 1265 if(success) 1266 { 1267 String result = doScrape(); 1268 JsonObject jsO = toJsonObject(result); 1269 JsonArray jsA = jsO.getJsonArray("Data"); 1270 dataHome = jsA.getJsonObject(0); 1271 holdings = dataHome.getJsonArray("Holdings"); 1272 } 1273 } 1274 else 1275 System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]"); 1276 1277 if(dataHome!=null && holdings!=null) 1278 { 1279 if(!resultSetOnly) 1280 { 1281 String friendlyName = dataHome.getString("FriendlyName"); 1282 1283 retVal+=" FriendlyName : "+friendlyName; 1284 retVal+="\n"; 1285 retVal+=" MarketValue : "+dataHome.getJsonNumber("MarketValue"); 1286 retVal+="\n"; 1287 retVal+=" TradeCash : "+dataHome.getJsonNumber("TradeCash"); 1288 retVal+="\n"; 1289 retVal+=" BookValue : "+dataHome.getJsonNumber("BookValue"); 1290 retVal+="\n"; 1291 retVal+=" ------------------ ------------------------"; 1292 retVal+="\n"; 1293 retVal+=" UnrealizedGainLoss : "+dataHome.getJsonNumber("UnrealizedGainLoss"); 1294 retVal+="\n"; 1295 retVal+=" UnrealizedGain% : "+ 1296 dfe.format(dataHome.getJsonNumber("UnrealizedGainLossPercent").doubleValue()*100.0); 1297 retVal+="\n"; 1298 retVal+="\n Holdings Summary\n - - - - - - - - - - - - -\n"; 1299 } 1300 1301 String currSymbol = ""; 1302 String priceStr = ""; 1303 String currencySymbol = ""; 1304 JsonObject stock = null; 1305 for(int i=0; i<holdings.size();i++) 1306 { 1307 stock = holdings.getJsonObject(i); 1308 if(stock!=null) 1309 { 1310 currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1); 1311 if(!resultSetOnly) 1312 { 1313 retVal+=" "+currSymbol+" : "+stock.getJsonNumber("Quantity").toString()+" @ "+ 1314 stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1)+ 1315 " = "+stock.getJsonNumber("MarketValue").toString(); 1316 retVal+="\n"; 1317 } 1318 else 1319 { 1320 if(right(stock.getJsonString("Price").toString() ,2).equals("U\"")) 1321 { 1322 priceStr = stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-2); 1323 currencySymbol = "US$"; 1324 } 1325 else 1326 { 1327 priceStr = stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1); 1328 currencySymbol = "C$"; 1329 } 1330 //stockName, symbol, CRExchangeSymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate 1331 retVal+=stock.getString("SymbolDescription").toString()+COL_DELIM+ 1332 currSymbol+COL_DELIM+ 1333 stock.getString("Exchange").toString()+COL_DELIM+ 1334 stock.getJsonNumber("Quantity").toString()+COL_DELIM+ 1335 priceStr+COL_DELIM+ 1336 currencySymbol+COL_DELIM+ 1337 stock.getJsonNumber("BookValue").toString()+COL_DELIM+ 1338 stock.getJsonNumber("MarketValue").toString()+COL_DELIM+ 1339 stock.getJsonNumber("UnrealizedGainLoss").toString()+COL_DELIM+ 1340 dfe.format(stock.getJsonNumber("UnrealizedGainLossPercent").doubleValue()*100.0)+COL_DELIM+ 1341 dateStr_; 1342 retVal+="\n"; 1343 } 1344 } 1345 } 1346 if(!resultSetOnly) retVal+="\n---------------------------\nHoldings Details:\n---------------------------\n"+ 1347 prettyJson(holdings.toString()); 1348 } 1349 1350 return retVal; 1351 } 1352 1353 1354 public String right(String value, int length) 1355 { 1356 // To get right characters from a string, change the begin index. 1357 return value.substring(value.length() - length); 1358 } 1359 1360 1361 /** Test method to do whatever tests I want. **/ 1362 @Override 1363 protected void test(String[] args) 1364 { 1365 super.debugOut_=true; 1366 debugOut_=true; 1367 1368 //NASDAQ:FCEL CVE:HIVE MUTF_CA:AGF201 TSE:LUN 1369 if(true) 1370 { 1371 String symbol = "AGF201"; 1372 String market = "MUTF_CA"; 1373 double result = getQuote(symbol,market); 1374 System.out.println("Test Quote For "+symbol+"="+result); 1375 //writeStringToFile(result,"gQuote-"+symbol+".txt"); 1376 } 1377 else if(false) 1378 { 1379 System.out.println("..."); 1380 try 1381 { 1382 1383 } 1384 catch (Exception ex) 1385 { 1386 System.out.println("NO GO..."); 1387 ex.printStackTrace(); 1388 } 1389 } 1390 } 1391 1392 1393 public static void main(String[] args) 1394 { 1395 QTradeScraper instance = new QTradeScraper(); 1396 1397 if(args.length>0 && args[0].toLowerCase().equals("-t")) 1398 instance.test(args); 1399 else if(args.length>1 && args[0].equalsIgnoreCase("quote")) 1400 { 1401 System.out.println("\n\nQuote For market:symbol "+args[1]+":"+args[2]+" = "+instance.getQuote(args[2],args[1])); 1402 System.out.println("Volume For market:symbol "+args[1]+":"+args[2]+" = "+instance.getVolume(args[2],args[1])); 1403 System.out.println("Open Price For market:symbol "+args[1]+":"+args[2]+" = "+instance.getOpen(args[2],args[1])); 1404 System.out.println("Days LOW Price For market:symbol "+args[1]+":"+args[2]+" = "+instance.getDaysLow(args[2],args[1])); 1405 System.out.println("Days High Price For market:symbol "+args[1]+":"+args[2]+" = "+instance.getDaysHigh(args[2],args[1])); 1406 System.out.println("Dividend Yield % For market:symbol "+args[1]+":"+args[2]+" = "+instance.getLatestDividend(args[2],args[1])); 1407 } 1408 else 1409 { 1410 boolean success = false; 1411 if(!"".equals(instance.getUsername()) && !"".equals(instance.getPassword())) 1412 success = instance.doLogin(); 1413 else 1414 System.out.println("ERROR reading username and password"); 1415 1416 if(success) 1417 { 1418 String result = instance.doScrape(); 1419 System.out.println("Saving Scraped page results to file: "+"gPageContent-"+instance.dateStr_+".json"); 1420 writeStringToFile(prettyJson(result),"gPageContent-"+instance.dateStr_+".json"); 1421 1422 System.out.println(instance.getPortfolioString()); 1423 } 1424 } 1425 } 1426}