refactor: scrapers

README.md

@@ -41,6 +41,7 @@
   * [.futopt.txoLargeTradersPosition(options)](#futopttxolargetraderspositionoptions)
   * [.futopt.exchangeRates(options)](#futoptexchangeratesoptions)
 * [Data Sources](#data-sources)
+* [Disclaimer](#disclaimer)
 * [Changelog](#changelog)
 * [License](#license)
 
@@ -1676,7 +1677,13 @@ twstock.futopt.exchangeRates({ date: '2023-01-30' })
 * [臺灣期貨交易所](https://www.taifex.com.tw)
 * [臺灣集中保管結算所](https://www.tdcc.com.tw)
 * [公開資訊觀測站](https://mops.twse.com.tw)
-* [基本市況報導網站](https://mis.twse.com.tw)
+* [臺灣證券交易所-基本市況報導網站](https://mis.twse.com.tw)
+* [臺灣期貨交易所行情資訊網](https://mis.taifex.com.tw)
+
+## Disclaimer
+
+- 透過本服務取得之資料僅供參考，若因任何資料之不正確或疏漏所衍生之損害或損失使用者需自行負責。本服務對資料內容錯誤、更新延誤或傳輸中斷不負任何責任。
+- 使用者應遵守臺灣證券交易所股份有限公司交易資訊使用管理辦法、臺灣期貨交易所股份有限公司交易資訊使用管理辦法、財團法人中華民國證券櫃檯買賣中心有價證券交易資訊使用管理辦法、各資訊來源提供者所定之資訊使用相關規範及智慧財產權相關法令。
 
 ## Changelog
 

src/scrapers/isin-scraper.ts

@@ -6,7 +6,7 @@ import { Scraper } from './scraper';
 import { asExchange, asMarket, asIndustry } from '../utils';
 
 export class IsinScraper extends Scraper {
-  async fetchStocksInfo(options: { symbol: string }) {
+  async fetchListed(options: { symbol: string }) {
     const { symbol } = options;
     const url = `https://isin.twse.com.tw/isin/single_main.jsp?owncode=${symbol}`;
     const response = await this.httpService.get(url, { responseType: 'arraybuffer' });
@@ -55,7 +55,7 @@ export class IsinScraper extends Scraper {
     return data;
   }
 
-  async fetchListedFutOpt() {
+  async fetchListedFutOpt(options?: { type?: 'F' | 'O'}) {
     const url = 'https://isin.twse.com.tw/isin/class_main.jsp?market=7';
     const response = await this.httpService.get(url, { responseType: 'arraybuffer' });
     const page = iconv.decode(response.data, 'big5');
@@ -74,6 +74,10 @@ export class IsinScraper extends Scraper {
       } as Record<string, any>;
     }).toArray();
 
-    return data;
+    return data.filter(row => {
+      if (options?.type === 'F') return row.type.includes('期貨');
+      if (options?.type === 'O') return row.type.includes('選擇權');
+      return true;
+    });
   }
 }

src/scrapers/mis-taifex-scraper.ts

@@ -19,13 +19,57 @@ export class MisTaifexScraper extends Scraper {
     const data = json.RtData.Items.map((row: any) => ({
       symbol: row.CID,
       name: row.DispCName,
+      type: type ?? this.extractTypeFromCmdyDDLItem(row),
       ...(row.SpotID && { underlying: row.SpotID }),
     }));
 
     return data;
   }
 
-  async fetchFutOptQuote(options: { ticker: Ticker, afterhours?: boolean }) {
+  async fetchFutOptQuoteList(options: { ticker: Ticker, afterhours?: boolean }) {
+    const { ticker, afterhours } = options;
+    const body = JSON.stringify({
+      CID: ticker.symbol,
+      SymbolType: ticker.type,
+      MarketType: afterhours ? 1 : 0,
+    });
+    const url = `https://mis.taifex.com.tw/futures/api/getQuoteList`;
+
+    const response = await this.httpService.post(url, body, {
+      headers: { 'Content-Type': 'application/json' },
+    });
+    const json = (response.data.RtCode === '0') && response.data;
+    if (!json) return null;
+
+    const data = json.RtData.QuoteList.map((row: any) => ({
+      symbol: row.SymbolID.split('-')[0],
+      name: row.DispCName,
+      status: row.Status,
+      openPrice: row.COpenPrice && numeral(row.COpenPrice).value(),
+      highPrice: row.CHighPrice && numeral(row.CHighPrice).value(),
+      lowPrice: row.CLowPrice && numeral(row.CLowPrice).value(),
+      lastPrice: row.CLastPrice && numeral(row.CLastPrice).value(),
+      referencePrice: row.CRefPrice && numeral(row.CRefPrice).value(),
+      limitUpPrice: row.CCeilPrice && numeral(row.CCeilPrice).value(),
+      limitDownPrice: row.CFloorPrice && numeral(row.CFloorPrice).value(),
+      settlementPrice: row.SettlementPrice && numeral(row.SettlementPrice).value(),
+      change: row.CDiff && numeral(row.CDiff).value(),
+      changePercent: row.CDiffRate && numeral(row.CDiffRate).value(),
+      amplitude: row.CAmpRate && numeral(row.CAmpRate).value(),
+      totalVoluem: row.CTotalVolume && numeral(row.CTotalVolume).value(),
+      openInterest: row.OpenInterest && numeral(row.OpenInterest).value(),
+      bestBidPrice: row.CBestBidPrice && numeral(row.CBestBidPrice).value(),
+      bestAskPrice: row.CBestAskPrice && numeral(row.CBestAskPrice).value(),
+      bestBidSize: row.CBestBidSize && numeral(row.CBestBidSize).value(),
+      bestAskSize: row.CBestAskSize && numeral(row.CBestAskSize).value(),
+      testPrice: row.CTestPrice && numeral(row.CTestPrice).value(),
+      testSize: row.CTestVolume && numeral(row.CTestVolume).value(),
+      lastUpdated: row.CTime && DateTime.fromFormat(`${row.CDate} ${row.CTime}`, 'yyyyMMdd hhmmss', { zone: 'Asia/Taipei' }).toMillis(),
+    })) as Record<string, any>[];
+    return data;
+  }
+
+  async fetchFutOptQuoteDetail(options: { ticker: Ticker, afterhours?: boolean }) {
     const { ticker, afterhours } = options;
     const body = JSON.stringify({ SymbolID: [this.extractSymbolIdFromTicker(ticker, { afterhours })] });
     const url = `https://mis.taifex.com.tw/futures/api/getQuoteDetail`;
@@ -38,7 +82,8 @@ export class MisTaifexScraper extends Scraper {
 
     const data = json.RtData.QuoteList.map((row: any) => ({
       symbol: ticker.symbol,
-      name: ticker.name,
+      name: row.DispCName,
+      status: row.Status,
       referencePrice: row.CRefPrice && numeral(row.CRefPrice).value(),
       limitUpPrice: row.CCeilPrice && numeral(row.CCeilPrice).value(),
       limitDownPrice: row.CFloorPrice && numeral(row.CFloorPrice).value(),
@@ -80,4 +125,11 @@ export class MisTaifexScraper extends Scraper {
       return afterhours ? `${symbol}-N` : `${symbol}-O`;
     }
   }
+
+  private extractTypeFromCmdyDDLItem(item: Record<string, any>) {
+    const type = item.CID.charAt(2);
+    if (['F', 'O'].includes(type)) return type;
+    if (item.DispCName.includes('期貨')) return 'F';
+    if (item.DispCName.includes('選擇權')) return 'O';
+  }
 }

src/twstock.ts

@@ -15,17 +15,17 @@ export class TwStock {
 
   get stocks() {
     return {
-      list: this.getStocksList.bind(this),
-      quote: this.getStocksQuote.bind(this),
-      historical: this.getStocksHistorical.bind(this),
-      instTrades: this.getStocksInstTrades.bind(this),
-      finiHoldings: this.getStocksFiniHoldings.bind(this),
-      marginTrades: this.getStocksMarginTrades.bind(this),
-      shortSales: this.getStocksShortSales.bind(this),
-      values: this.getStocksValues.bind(this),
-      holders: this.getStocksHolders.bind(this),
-      eps: this.getStocksEps.bind(this),
-      revenue: this.getStocksRevenue.bind(this),
+      list: this.fetchStocksList.bind(this),
+      quote: this.fetchStocksQuote.bind(this),
+      historical: this.fetchStocksHistorical.bind(this),
+      instTrades: this.fetchStocksInstTrades.bind(this),
+      finiHoldings: this.fetchStocksFiniHoldings.bind(this),
+      marginTrades: this.fetchStocksMarginTrades.bind(this),
+      shortSales: this.fetchStocksShortSales.bind(this),
+      values: this.fetchStocksValues.bind(this),
+      holders: this.fetchStocksHolders.bind(this),
+      eps: this.fetchStocksEps.bind(this),
+      revenue: this.fetchStocksRevenue.bind(this),
     };
   }
 
@@ -51,6 +51,7 @@ export class TwStock {
     return {
       list: this.getFutOptList.bind(this),
       quote: this.getFutOptQuote.bind(this),
+      contracts: this.getFutOptContracts.bind(this),
       historical: this.getFutOptHistorical.bind(this),
       txfInstTrades: this.getFutOptTxfInstTrades.bind(this),
       txoInstTrades: this.getFutOptTxoInstTrades.bind(this),
@@ -67,7 +68,7 @@ export class TwStock {
     const isinScraper = this._scraper.getIsinScraper();
 
     const stocks = (symbol)
-      ? await isinScraper.fetchStocksInfo({ symbol })
+      ? await isinScraper.fetchListed({ symbol })
       : await Promise.all([
           isinScraper.fetchListedStocks({ market: Market.TSE }),
           isinScraper.fetchListedStocks({ market: Market.OTC }),
@@ -97,26 +98,43 @@ export class TwStock {
     return indices;
   }
 
-  private async loadFutOpt(options?: { symbol?: string }) {
-    const { symbol } = options ?? {};
+  private async loadFutOpt(options?: { type?: 'F' | 'O' }) {
+    const { type } = options ?? {};
+    const misTaifexScraper = this._scraper.getMisTaifexScraper();
+
+    const futopt = await (() => {
+      switch (type) {
+        case 'F': return misTaifexScraper.fetchListedFutOpt({ type: 'F' });
+        case 'O': return misTaifexScraper.fetchListedFutOpt({ type: 'O' });
+        default: return misTaifexScraper.fetchListedFutOpt();
+      }
+    })() as Ticker[];
+
+    futopt.forEach(({ symbol, ...ticker }) => this._futopt.set(symbol, { symbol, ...ticker }));
+
+    return futopt;
+  }
+
+  private async loadFutOptContracts(options?: { symbol?: string, type?: 'F' | 'O' }) {
+    const { symbol, type } = options ?? {};
     const isinScraper = this._scraper.getIsinScraper();
 
     const futopt = (symbol)
-      ? await isinScraper.fetchStocksInfo({ symbol })
-      : await isinScraper.fetchListedFutOpt();
+      ? await isinScraper.fetchListed({ symbol })
+      : await isinScraper.fetchListedFutOpt({ type });
 
     futopt.forEach(({ symbol, ...ticker }) => this._futopt.set(symbol, { symbol, ...ticker }));
 
     return futopt;
   }
 
-  private async getStocksList(options?: { market: 'TSE' | 'OTC' }) {
+  private async fetchStocksList(options?: { market: 'TSE' | 'OTC' }) {
     const { market } = options ?? {};
     const data = await this.loadStocks();
     return market ? data.filter(ticker => ticker.market === market) : data;
   }
 
-  private async getStocksQuote(options: { symbol: string, odd?: boolean }) {
+  private async fetchStocksQuote(options: { symbol: string, odd?: boolean }) {
     const { symbol, odd } = options;
 
     if (!this._stocks.has(symbol)) {
@@ -128,7 +146,7 @@ export class TwStock {
     return this._scraper.getMisTwseScraper().fetchStocksQuote({ ticker, odd });
   }
 
-  private async getStocksHistorical(options: { date: string, market?: 'TSE' | 'OTC', symbol?: string }) {
+  private async fetchStocksHistorical(options: { date: string, market?: 'TSE' | 'OTC', symbol?: string }) {
     const { date, symbol } = options;
 
     if (symbol && !this._stocks.has(symbol)) {
@@ -144,7 +162,7 @@ export class TwStock {
       : await this._scraper.getTwseScraper().fetchStocksHistorical({ date, symbol });
   }
 
-  private async getStocksInstTrades(options: { date: string, market?: 'TSE' | 'OTC', symbol?: string }) {
+  private async fetchStocksInstTrades(options: { date: string, market?: 'TSE' | 'OTC', symbol?: string }) {
     const { date, symbol } = options;
 
     if (symbol && !this._stocks.has(symbol)) {
@@ -160,7 +178,7 @@ export class TwStock {
       : await this._scraper.getTwseScraper().fetchStocksInstInvestorsTrades({ date, symbol });
   }
 
-  private async getStocksFiniHoldings(options: { date: string, market?: 'TSE' | 'OTC', symbol?: string }) {
+  private async fetchStocksFiniHoldings(options: { date: string, market?: 'TSE' | 'OTC', symbol?: string }) {
     const { date, symbol } = options;
 
     if (symbol && !this._stocks.has(symbol)) {
@@ -176,7 +194,7 @@ export class TwStock {
       : await this._scraper.getTwseScraper().fetchStocksFiniHoldings({ date, symbol });
   }
 
-  private async getStocksMarginTrades(options: { date: string, market?: 'TSE' | 'OTC', symbol?: string }) {
+  private async fetchStocksMarginTrades(options: { date: string, market?: 'TSE' | 'OTC', symbol?: string }) {
     const { date, symbol } = options;
 
     if (symbol && !this._stocks.has(symbol)) {
@@ -192,7 +210,7 @@ export class TwStock {
       : await this._scraper.getTwseScraper().fetchStocksMarginTrades({ date, symbol });
   }
 
-  private async getStocksShortSales(options: { date: string, market?: 'TSE' | 'OTC', symbol?: string }) {
+  private async fetchStocksShortSales(options: { date: string, market?: 'TSE' | 'OTC', symbol?: string }) {
     const { date, symbol } = options;
 
     if (symbol && !this._stocks.has(symbol)) {
@@ -208,7 +226,7 @@ export class TwStock {
       : await this._scraper.getTwseScraper().fetchStocksShortSales({ date, symbol });
   }
 
-  private async getStocksValues(options: { date: string, market?: 'TSE' | 'OTC', symbol?: string }) {
+  private async fetchStocksValues(options: { date: string, market?: 'TSE' | 'OTC', symbol?: string }) {
     const { date, symbol } = options;
 
     if (symbol && !this._stocks.has(symbol)) {
@@ -224,7 +242,7 @@ export class TwStock {
       : await this._scraper.getTwseScraper().fetchStocksValues({ date, symbol });
   }
 
-  private async getStocksHolders(options: { date: string, symbol: string }) {
+  private async fetchStocksHolders(options: { date: string, symbol: string }) {
     const { date, symbol } = options;
 
     if (symbol && !this._stocks.has(symbol)) {
@@ -235,7 +253,7 @@ export class TwStock {
     return this._scraper.getTdccScraper().fetchStocksHolders({ date, symbol });
   }
 
-  private async getStocksEps(options: { year: number, quarter: number, market?: 'TSE' | 'OTC', symbol?: string }) {
+  private async fetchStocksEps(options: { year: number, quarter: number, market?: 'TSE' | 'OTC', symbol?: string }) {
     const { symbol, year, quarter } = options;
 
     if (!options.market && !options.symbol) {
@@ -255,7 +273,7 @@ export class TwStock {
     return this._scraper.getMopsScraper().fetchStocksEps({ market, year, quarter, symbol });
   }
 
-  private async getStocksRevenue(options: { market?: 'TSE' | 'OTC', year: number, month: number, foreign?: boolean, symbol?: string  }) {
+  private async fetchStocksRevenue(options: { market?: 'TSE' | 'OTC', year: number, month: number, foreign?: boolean, symbol?: string  }) {
     const { symbol, year, month, foreign } = options;
 
     if (symbol && !this._stocks.has(symbol)) {
@@ -352,21 +370,34 @@ export class TwStock {
       : await this._scraper.getTwseScraper().fetchMarketMarginTrades({ date });
   }
 
-  private async getFutOptList() {
-    const data = await this.loadFutOpt();
+  private async getFutOptList(options?: { type?: 'F' | 'O' }) {
+    const { type } = options ?? {};
+    const data = await this.loadFutOpt({ type });
+    return data;
+  }
+
+  private async getFutOptContracts(options?: { type?: 'F' | 'O' }) {
+    const { type } = options ?? {};
+    const data = await this.loadFutOptContracts({ type });
     return data;
   }
 
   private async getFutOptQuote(options: { symbol: string, afterhours?: boolean }) {
     const { symbol, afterhours } = options;
 
     if (!this._futopt.has(symbol)) {
-      const futopt = await this.loadFutOpt({ symbol });
-      if (!futopt.length) throw new Error('symbol not found');
+      const futopt = (symbol.length === 3)
+        ? await this.loadFutOpt()
+        : await this.loadFutOptContracts({ symbol });
+
+      if (!map(futopt, 'symbol').includes(symbol)) throw new Error('symbol not found');
     }
 
     const ticker = this._futopt.get(symbol) as Ticker;
-    return this._scraper.getMisTaifexScraper().fetchFutOptQuote({ ticker, afterhours });
+
+    return (ticker.symbol.length === 3)
+      ? this._scraper.getMisTaifexScraper().fetchFutOptQuoteList({ ticker, afterhours })
+      : this._scraper.getMisTaifexScraper().fetchFutOptQuoteDetail({ ticker, afterhours });
   }
 
   private async getFutOptHistorical(options: { date: string, symbol: string, afterhours?: boolean }) {

test/fixtures/fetched-taifex-options-list.json

@@ -0,0 +1 @@
+[{"symbol":"TXO","name":"臺指選","type":"O"},{"symbol":"TFO","name":"金指選","type":"O"},{"symbol":"TEO","name":"電指選","type":"O"},{"symbol":"TGO","name":"黃金選","type":"O"},{"symbol":"NYO","name":"元大台灣50ETF選擇權","type":"O","underlying":"0050"},{"symbol":"OAO","name":"富邦上証ETF選擇權","type":"O","underlying":"006205"},{"symbol":"OBO","name":"元大上證50ETF選擇權","type":"O","underlying":"006206"},{"symbol":"OJO","name":"國泰中國A50ETF選擇權","type":"O","underlying":"00636"},{"symbol":"OKO","name":"富邦深100ETF選擇權","type":"O","underlying":"00639"},{"symbol":"OOO","name":"群益深証中小ETF選擇權","type":"O","underlying":"00643"},{"symbol":"DFO","name":"台泥選擇權","type":"O","underlying":"1101"},{"symbol":"CQO","name":"統一選擇權","type":"O","underlying":"1216"},{"symbol":"CFO","name":"台塑選擇權","type":"O","underlying":"1301"},{"symbol":"CAO","name":"南亞選擇權","type":"O","underlying":"1303"},{"symbol":"DGO","name":"台化選擇權","type":"O","underlying":"1326"},{"symbol":"CRO","name":"遠東新選擇權","type":"O","underlying":"1402"},{"symbol":"CSO","name":"華新選擇權","type":"O","underlying":"1605"},{"symbol":"CBO","name":"中鋼選擇權","type":"O","underlying":"2002"},{"symbol":"CCO","name":"聯電選擇權","type":"O","underlying":"2303"},{"symbol":"DHO","name":"鴻海選擇權","type":"O","underlying":"2317"},{"symbol":"CGO","name":"仁寶選擇權","type":"O","underlying":"2324"},{"symbol":"CDO","name":"台積電選擇權","type":"O","underlying":"2330"},{"symbol":"CDA","name":"台積電選擇權A","type":"O","underlying":"2330"},{"symbol":"DSO","name":"宏碁選擇權","type":"O","underlying":"2353"},{"symbol":"DJO","name":"華碩選擇權","type":"O","underlying":"2357"},{"symbol":"GIO","name":"微星選擇權","type":"O","underlying":"2377"},{"symbol":"DKO","name":"廣達選擇權","type":"O","underlying":"2382"},{"symbol":"CHO","name":"友達選擇權","type":"O","underlying":"2409"},{"symbol":"DLO","name":"中華電選擇權","type":"O","underlying":"2412"},{"symbol":"DVO","name":"聯發科選擇權","type":"O","underlying":"2454"},{"symbol":"DVA","name":"聯發科選擇權A","type":"O","underlying":"2454"},{"symbol":"GXO","name":"可成選擇權","type":"O","underlying":"2474"},{"symbol":"HCO","name":"宏達電選擇權","type":"O","underlying":"2498"},{"symbol":"CZO","name":"長榮選擇權","type":"O","underlying":"2603"},{"symbol":"HSO","name":"長榮航選擇權","type":"O","underlying":"2618"},{"symbol":"DCO","name":"彰銀選擇權","type":"O","underlying":"2801"},{"symbol":"CJO","name":"華南金選擇權","type":"O","underlying":"2880"},{"symbol":"CEO","name":"富邦金選擇權","type":"O","underlying":"2881"},{"symbol":"CKO","name":"國泰金選擇權","type":"O","underlying":"2882"},{"symbol":"DNO","name":"玉山金選擇權","type":"O","underlying":"2884"},{"symbol":"DOO","name":"元大金選擇權","type":"O","underlying":"2885"},{"symbol":"CLO","name":"兆豐金選擇權","type":"O","underlying":"2886"},{"symbol":"CMO","name":"台新金選擇權","type":"O","underlying":"2887"},{"symbol":"DDO","name":"新光金選擇權","type":"O","underlying":"2888"},{"symbol":"DEO","name":"永豐金選擇權","type":"O","underlying":"2890"},{"symbol":"CNO","name":"中信金選擇權","type":"O","underlying":"2891"},{"symbol":"DPO","name":"第一金選擇權","type":"O","underlying":"2892"},{"symbol":"IJO","name":"大立光選擇權","type":"O","underlying":"3008"},{"symbol":"IRO","name":"欣興選擇權","type":"O","underlying":"3037"},{"symbol":"DXO","name":"緯創選擇權","type":"O","underlying":"3231"},{"symbol":"DQO","name":"群創選擇權","type":"O","underlying":"3481"},{"symbol":"OZO","name":"日月光投控選擇權","type":"O","underlying":"3711"},{"symbol":"QBO","name":"富采選擇權","type":"O","underlying":"3714"},{"symbol":"LOO","name":"合庫金選擇權","type":"O","underlying":"5880"}]