feat(rubick): rename project and add production release bundle

This commit is contained in:
Muhammad Firas 2026-03-06 03:12:28 +07:00
commit fab6cf26eb
No known key found for this signature in database
33 changed files with 11554 additions and 0 deletions

31
scripts/e2e_run.sh Executable file
View file

@ -0,0 +1,31 @@
#!/usr/bin/env bash
set -euo pipefail
APP_BIN="${1:-./bin/rubick}"
if [[ ! -x "$APP_BIN" ]]; then
echo "error: binary not executable: $APP_BIN" >&2
exit 1
fi
TS="$(date +%Y%m%d-%H%M%S)"
OUT="output/$TS"
mkdir -p "$OUT"
echo "timestamp=$TS" > "$OUT/RUN_INFO.txt"
echo "[1/5] msn fetch-all"
"$APP_BIN" msn fetch-all --index idx30 --limit 3 --db "$OUT/stocks.db" --rps 10 --delay 100-150 --concurrency 2
echo "[2/5] news plain"
"$APP_BIN" news IHSG --from 2026-03-01 --to 2026-03-05 --count 2 --concurrency 2 --output "$OUT/news_plain.json"
echo "[3/5] export simple csv"
"$APP_BIN" export simple --db "$OUT/stocks.db" --format csv --output "$OUT/csv"
echo "[4/5] export dashboard"
"$APP_BIN" export dashboard --db "$OUT/stocks.db" --output "$OUT/dashboard.xlsx"
echo "[5/5] export history"
"$APP_BIN" export history --db "$OUT/stocks.db" --output "$OUT/history.xlsx"
echo "E2E completed: $OUT"

1373
scripts/export_dashboard.py Normal file

File diff suppressed because it is too large Load diff

626
scripts/export_history.py Normal file
View file

@ -0,0 +1,626 @@
#!/usr/bin/env python3
"""
History Excel Export Script - Professional Edition
Exports historical data (price history, ratio history, sentiment history) to Excel
with proper tables and conditional formatting. No charts - tables only.
Usage:
uv run python scripts/export_history.py --db output/stocks.db --output output/history.xlsx
"""
import argparse
import sqlite3
from datetime import datetime
from pathlib import Path
from typing import Optional
import xlsxwriter
from xlsxwriter.utility import xl_range
# Color palette - matches export_dashboard.py
COLORS = {
'primary_dark': '#1F4E79',
'primary': '#2E75B6',
'primary_light': '#5B9BD5',
'success_dark': '#375623',
'success': '#70AD47',
'success_light': '#C6EFCE',
'success_text': '#006100',
'danger_dark': '#833C0C',
'danger': '#C00000',
'danger_light': '#FFC7CE',
'danger_text': '#9C0006',
'warning_dark': '#7F6000',
'warning': '#FFC000',
'warning_light': '#FFEB9C',
'warning_text': '#9C5700',
'white': '#FFFFFF',
'light_gray': '#F2F2F2',
'dark_gray': '#404040',
}
TABLE_STYLE = 'Table Style Medium 2'
def safe_float(value, default=None) -> Optional[float]:
"""Safely convert to float."""
if value is None:
return default
try:
return float(value)
except (ValueError, TypeError):
return default
class HistoryExporter:
"""Professional History Excel Exporter."""
def __init__(self, workbook: xlsxwriter.Workbook, conn: sqlite3.Connection,
start_date: str = None, end_date: str = None, stock_id: str = None):
self.wb = workbook
self.conn = conn
self.start_date = start_date
self.end_date = end_date
self.stock_id = stock_id
self._setup_formats()
def _setup_formats(self):
"""Setup formatting styles - matches export_dashboard.py."""
self.fmt_title = self.wb.add_format({
'bold': True, 'font_size': 18, 'font_color': COLORS['primary_dark'],
'bottom': 2, 'bottom_color': COLORS['primary_dark']
})
self.fmt_subtitle = self.wb.add_format({
'bold': True, 'font_size': 12, 'font_color': COLORS['primary']
})
self.fmt_section = self.wb.add_format({
'bold': True, 'font_size': 14, 'font_color': COLORS['white'],
'bg_color': COLORS['primary_dark'], 'align': 'center', 'valign': 'vcenter'
})
self.fmt_header = self.wb.add_format({
'bold': True, 'font_color': COLORS['white'],
'bg_color': COLORS['primary_dark'], 'align': 'center', 'valign': 'vcenter',
'border': 1, 'text_wrap': True
})
self.fmt_num = self.wb.add_format({'num_format': '#,##0.00', 'align': 'right'})
self.fmt_num_0 = self.wb.add_format({'num_format': '#,##0', 'align': 'right'})
self.fmt_pct = self.wb.add_format({'num_format': '0.00"%"', 'align': 'right'})
self.fmt_pct_signed = self.wb.add_format({'num_format': '+0.00%;-0.00%;0.00%', 'align': 'right'})
self.fmt_date = self.wb.add_format({'num_format': 'yyyy-mm-dd', 'align': 'center'})
self.fmt_kpi_value = self.wb.add_format({
'bold': True, 'font_size': 24, 'font_color': COLORS['primary_dark'],
'align': 'center', 'valign': 'vcenter'
})
self.fmt_kpi_label = self.wb.add_format({
'font_size': 10, 'font_color': COLORS['dark_gray'],
'align': 'center', 'valign': 'vcenter', 'bold': True
})
self.fmt_kpi_box = self.wb.add_format({
'bg_color': COLORS['light_gray'], 'border': 1, 'border_color': COLORS['primary_light']
})
def _add_table(self, ws, start_row: int, start_col: int, data: list,
columns: list, table_name: str, total_row: bool = False) -> int:
"""Add a proper Excel Table with filtering and sorting."""
if not data:
ws.write(start_row, start_col, "No data available")
return start_row + 1
end_row = start_row + len(data)
end_col = start_col + len(columns) - 1
# Build table columns
table_columns = []
for col_def in columns:
col_opt = {'header': col_def['header']}
if col_def.get('total_function'):
col_opt['total_function'] = col_def['total_function']
if col_def.get('total_string'):
col_opt['total_string'] = col_def['total_string']
if col_def.get('format'):
col_opt['format'] = col_def['format']
table_columns.append(col_opt)
# Write data
for row_idx, row_data in enumerate(data):
for col_idx, col_def in enumerate(columns):
key = col_def.get('key')
value = row_data.get(key) if key else None
transform = col_def.get('transform')
if transform and value is not None:
value = transform(value)
fmt = col_def.get('format')
ws.write(start_row + 1 + row_idx, start_col + col_idx, value, fmt)
# Add table
table_range = xl_range(start_row, start_col, end_row, end_col)
ws.add_table(table_range, {
'name': table_name,
'style': TABLE_STYLE,
'columns': table_columns,
'total_row': total_row,
'autofilter': True,
})
# Set column widths
for col_idx, col_def in enumerate(columns):
width = col_def.get('width', 12)
ws.set_column(start_col + col_idx, start_col + col_idx, width)
return end_row + (2 if total_row else 1)
def _get_stocks(self) -> list:
"""Get all stocks."""
cursor = self.conn.execute("""
SELECT id, ticker, name, exchange_code as exchange
FROM stocks ORDER BY ticker
""")
return [dict(row) for row in cursor.fetchall()]
def _get_price_history(self) -> list:
"""Get price history with filters."""
query = """
SELECT ph.*, s.ticker, s.name
FROM price_history ph
JOIN stocks s ON ph.stock_id = s.id
WHERE 1=1
"""
params = []
if self.stock_id:
query += " AND ph.stock_id = ?"
params.append(self.stock_id)
if self.start_date:
query += " AND ph.scrape_date >= ?"
params.append(self.start_date)
if self.end_date:
query += " AND ph.scrape_date <= ?"
params.append(self.end_date)
query += " ORDER BY s.ticker, ph.scrape_date DESC"
return [dict(row) for row in self.conn.execute(query, params).fetchall()]
def _get_ratios_history(self) -> list:
"""Get ratios history with filters."""
query = """
SELECT rh.*, s.ticker, s.name
FROM ratios_history rh
JOIN stocks s ON rh.stock_id = s.id
WHERE 1=1
"""
params = []
if self.stock_id:
query += " AND rh.stock_id = ?"
params.append(self.stock_id)
if self.start_date:
query += " AND rh.scrape_date >= ?"
params.append(self.start_date)
if self.end_date:
query += " AND rh.scrape_date <= ?"
params.append(self.end_date)
query += " ORDER BY s.ticker, rh.scrape_date DESC"
return [dict(row) for row in self.conn.execute(query, params).fetchall()]
def _get_sentiment_history(self) -> list:
"""Get sentiment history with filters."""
query = """
SELECT sh.*, s.ticker, s.name
FROM sentiment_history sh
JOIN stocks s ON sh.stock_id = s.id
WHERE 1=1
"""
params = []
if self.stock_id:
query += " AND sh.stock_id = ?"
params.append(self.stock_id)
if self.start_date:
query += " AND sh.scrape_date >= ?"
params.append(self.start_date)
if self.end_date:
query += " AND sh.scrape_date <= ?"
params.append(self.end_date)
query += " ORDER BY s.ticker, sh.scrape_date DESC"
return [dict(row) for row in self.conn.execute(query, params).fetchall()]
def _get_scrape_runs(self) -> list:
"""Get scrape runs."""
cursor = self.conn.execute("""
SELECT * FROM scrape_runs ORDER BY started_at DESC
""")
return [dict(row) for row in cursor.fetchall()]
def create_summary_sheet(self, stocks: list, price_history: list,
ratios_history: list, sentiment_history: list,
scrape_runs: list):
"""Create summary sheet."""
ws = self.wb.add_worksheet("Summary")
# Title
ws.merge_range('A1:F1', "Historical Data Summary", self.fmt_title)
ws.set_row(0, 30)
ws.write('A2', f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M')}", self.fmt_subtitle)
# KPI cards
row = 4
kpis = [
("Total Stocks", str(len(stocks))),
("Price Records", str(len(price_history))),
("Ratio Records", str(len(ratios_history))),
("Sentiment Records", str(len(sentiment_history))),
("Scrape Runs", str(len(scrape_runs))),
]
col = 0
for label, value in kpis:
ws.merge_range(row, col, row + 1, col + 1, '', self.fmt_kpi_box)
ws.write(row, col, label, self.fmt_kpi_label)
ws.write(row + 1, col, value, self.fmt_kpi_value)
col += 2
# Date range
row = 7
ws.write(row, 0, "Data Date Range:", self.fmt_subtitle)
if price_history:
dates = [p['scrape_date'] for p in price_history if p.get('scrape_date')]
if dates:
ws.write(row + 1, 0, f"From: {min(dates)}")
ws.write(row + 2, 0, f"To: {max(dates)}")
# Scrape runs summary table
row = 11
ws.merge_range(row, 0, row, 5, "Recent Scrape Runs", self.fmt_section)
row += 1
run_data = []
for r in scrape_runs[:10]:
started = r.get('started_at', '')
finished = r.get('finished_at', '')
duration = ''
if started and finished:
try:
start_dt = datetime.fromisoformat(started.replace('Z', '+00:00'))
finish_dt = datetime.fromisoformat(finished.replace('Z', '+00:00'))
delta = finish_dt - start_dt
duration = str(delta)
except:
pass
run_data.append({
'id': r.get('id'),
'status': r.get('status'),
'index': r.get('index_name'),
'total': r.get('total_stocks'),
'success': r.get('success'),
'failed': r.get('failed'),
'duration': duration,
})
run_cols = [
{'header': 'Run ID', 'key': 'id', 'width': 8},
{'header': 'Status', 'key': 'status', 'width': 12},
{'header': 'Index', 'key': 'index', 'width': 10},
{'header': 'Total', 'key': 'total', 'width': 8, 'format': self.fmt_num_0},
{'header': 'Success', 'key': 'success', 'width': 8, 'format': self.fmt_num_0},
{'header': 'Failed', 'key': 'failed', 'width': 8, 'format': self.fmt_num_0},
{'header': 'Duration', 'key': 'duration', 'width': 15},
]
self._add_table(ws, row, 0, run_data, run_cols, 'ScrapeRuns')
ws.set_column('A:G', 12)
def create_price_history_sheet(self, data: list):
"""Create price history sheet."""
ws = self.wb.add_worksheet("Price History")
ws.merge_range('A1:N1', "Price History", self.fmt_title)
ws.set_row(0, 25)
row = 3
price_data = [{
'ticker': p['ticker'],
'name': p['name'],
'date': p.get('scrape_date', ''),
'price': safe_float(p.get('price')),
'change': safe_float(p.get('price_change')),
'change_pct': safe_float(p.get('price_change_pct'), 0) / 100 if p.get('price_change_pct') else None,
'open': safe_float(p.get('price_open')),
'high': safe_float(p.get('price_high')),
'low': safe_float(p.get('price_low')),
'volume': safe_float(p.get('volume')),
'market_cap': safe_float(p.get('market_cap')),
'high_52w': safe_float(p.get('price_52w_high')),
'low_52w': safe_float(p.get('price_52w_low')),
'ytd_pct': safe_float(p.get('price_change_ytd'), 0) / 100 if p.get('price_change_ytd') else None,
} for p in data]
price_cols = [
{'header': 'Ticker', 'key': 'ticker', 'width': 8},
{'header': 'Name', 'key': 'name', 'width': 22},
{'header': 'Date', 'key': 'date', 'width': 11},
{'header': 'Price', 'key': 'price', 'width': 10, 'format': self.fmt_num},
{'header': 'Change', 'key': 'change', 'width': 10, 'format': self.fmt_num},
{'header': 'Chg%', 'key': 'change_pct', 'width': 8, 'format': self.fmt_pct_signed},
{'header': 'Open', 'key': 'open', 'width': 10, 'format': self.fmt_num},
{'header': 'High', 'key': 'high', 'width': 10, 'format': self.fmt_num},
{'header': 'Low', 'key': 'low', 'width': 10, 'format': self.fmt_num},
{'header': 'Volume', 'key': 'volume', 'width': 14, 'format': self.fmt_num_0},
{'header': 'Market Cap', 'key': 'market_cap', 'width': 15, 'format': self.fmt_num_0},
{'header': '52W High', 'key': 'high_52w', 'width': 10, 'format': self.fmt_num},
{'header': '52W Low', 'key': 'low_52w', 'width': 10, 'format': self.fmt_num},
{'header': 'YTD%', 'key': 'ytd_pct', 'width': 8, 'format': self.fmt_pct_signed},
]
self._add_table(ws, row, 0, price_data, price_cols, 'PriceHistory')
# Conditional formatting on change %
if price_data:
data_end = row + len(price_data)
ws.conditional_format(row + 1, 5, data_end, 5, {
'type': '3_color_scale',
'min_color': '#F8696B',
'mid_color': '#FFEB84',
'max_color': '#63BE7B',
})
ws.freeze_panes(4, 2)
def create_ratios_history_sheet(self, data: list):
"""Create ratios history sheet."""
ws = self.wb.add_worksheet("Ratios History")
ws.merge_range('A1:W1', "Financial Ratios History", self.fmt_title)
ws.set_row(0, 25)
row = 3
ratio_data = [{
'ticker': r['ticker'],
'name': r['name'],
'date': r.get('scrape_date', ''),
'year': r.get('year', ''),
'pe': safe_float(r.get('pe_ratio')),
'pb': safe_float(r.get('pb_ratio')),
'ps': safe_float(r.get('ps_ratio')),
'pcf': safe_float(r.get('pcf_ratio')),
'ev_ebitda': safe_float(r.get('ev_ebitda')),
'roe': safe_float(r.get('roe')),
'roa': safe_float(r.get('roa')),
'roic': safe_float(r.get('roic')),
'gross': safe_float(r.get('gross_margin')),
'op_margin': safe_float(r.get('operating_margin')),
'net_margin': safe_float(r.get('net_margin')),
'de': safe_float(r.get('debt_to_equity')),
'current': safe_float(r.get('current_ratio')),
'quick': safe_float(r.get('quick_ratio')),
'div_yield': safe_float(r.get('dividend_yield')),
'payout': safe_float(r.get('payout_ratio')),
'eps': safe_float(r.get('eps')),
'bvps': safe_float(r.get('bvps')),
'rev_gr': safe_float(r.get('revenue_growth')),
} for r in data]
ratio_cols = [
{'header': 'Ticker', 'key': 'ticker', 'width': 8},
{'header': 'Name', 'key': 'name', 'width': 20},
{'header': 'Date', 'key': 'date', 'width': 11},
{'header': 'Year', 'key': 'year', 'width': 6},
{'header': 'P/E', 'key': 'pe', 'width': 7, 'format': self.fmt_num},
{'header': 'P/B', 'key': 'pb', 'width': 7, 'format': self.fmt_num},
{'header': 'P/S', 'key': 'ps', 'width': 7, 'format': self.fmt_num},
{'header': 'P/CF', 'key': 'pcf', 'width': 7, 'format': self.fmt_num},
{'header': 'EV/EBITDA', 'key': 'ev_ebitda', 'width': 9, 'format': self.fmt_num},
{'header': 'ROE%', 'key': 'roe', 'width': 7, 'format': self.fmt_num},
{'header': 'ROA%', 'key': 'roa', 'width': 7, 'format': self.fmt_num},
{'header': 'ROIC%', 'key': 'roic', 'width': 7, 'format': self.fmt_num},
{'header': 'Gross%', 'key': 'gross', 'width': 8, 'format': self.fmt_num},
{'header': 'Op%', 'key': 'op_margin', 'width': 7, 'format': self.fmt_num},
{'header': 'Net%', 'key': 'net_margin', 'width': 7, 'format': self.fmt_num},
{'header': 'D/E', 'key': 'de', 'width': 7, 'format': self.fmt_num},
{'header': 'Current', 'key': 'current', 'width': 8, 'format': self.fmt_num},
{'header': 'Quick', 'key': 'quick', 'width': 7, 'format': self.fmt_num},
{'header': 'Yield%', 'key': 'div_yield', 'width': 7, 'format': self.fmt_num},
{'header': 'Payout%', 'key': 'payout', 'width': 8, 'format': self.fmt_num},
{'header': 'EPS', 'key': 'eps', 'width': 8, 'format': self.fmt_num},
{'header': 'BVPS', 'key': 'bvps', 'width': 9, 'format': self.fmt_num},
{'header': 'RevGr%', 'key': 'rev_gr', 'width': 8, 'format': self.fmt_num},
]
self._add_table(ws, row, 0, ratio_data, ratio_cols, 'RatiosHistory')
# Conditional formatting on ROE
if ratio_data:
data_end = row + len(ratio_data)
ws.conditional_format(row + 1, 9, data_end, 9, {
'type': 'data_bar',
'bar_color': COLORS['success'],
'bar_solid': True,
})
ws.freeze_panes(4, 2)
def create_sentiment_history_sheet(self, data: list):
"""Create sentiment history sheet."""
ws = self.wb.add_worksheet("Sentiment History")
ws.merge_range('A1:K1', "Sentiment History", self.fmt_title)
ws.set_row(0, 25)
row = 3
sent_data = [{
'ticker': s['ticker'],
'name': s['name'],
'date': s.get('scrape_date', ''),
'time_range': s.get('time_range', ''),
'bullish': safe_float(s.get('bullish_pct')),
'bearish': safe_float(s.get('bearish_pct')),
'neutral': safe_float(s.get('neutral_pct')),
'bull_count': safe_float(s.get('bullish')),
'bear_count': safe_float(s.get('bearish')),
'neut_count': safe_float(s.get('neutral')),
'net': (safe_float(s.get('bullish_pct'), 0) - safe_float(s.get('bearish_pct'), 0)),
} for s in data]
sent_cols = [
{'header': 'Ticker', 'key': 'ticker', 'width': 8},
{'header': 'Name', 'key': 'name', 'width': 22},
{'header': 'Date', 'key': 'date', 'width': 11},
{'header': 'Period', 'key': 'time_range', 'width': 15},
{'header': 'Bull%', 'key': 'bullish', 'width': 8, 'format': self.fmt_num},
{'header': 'Bear%', 'key': 'bearish', 'width': 8, 'format': self.fmt_num},
{'header': 'Neut%', 'key': 'neutral', 'width': 8, 'format': self.fmt_num},
{'header': 'Bulls', 'key': 'bull_count', 'width': 8, 'format': self.fmt_num_0},
{'header': 'Bears', 'key': 'bear_count', 'width': 8, 'format': self.fmt_num_0},
{'header': 'Neutral', 'key': 'neut_count', 'width': 8, 'format': self.fmt_num_0},
{'header': 'Net', 'key': 'net', 'width': 8, 'format': self.fmt_num},
]
self._add_table(ws, row, 0, sent_data, sent_cols, 'SentimentHistory')
# Conditional formatting
if sent_data:
data_end = row + len(sent_data)
# Bullish data bar
ws.conditional_format(row + 1, 4, data_end, 4, {
'type': 'data_bar',
'bar_color': COLORS['success'],
'bar_solid': True,
})
# Bearish data bar
ws.conditional_format(row + 1, 5, data_end, 5, {
'type': 'data_bar',
'bar_color': COLORS['danger'],
'bar_solid': True,
})
# Net sentiment 3-color
ws.conditional_format(row + 1, 10, data_end, 10, {
'type': '3_color_scale',
'min_color': '#F8696B',
'mid_color': '#FFEB84',
'max_color': '#63BE7B',
})
ws.freeze_panes(4, 2)
def create_price_pivot_sheet(self, stocks: list, price_history: list):
"""Create price pivot table (dates as rows, stocks as columns)."""
ws = self.wb.add_worksheet("Price Pivot")
ws.merge_range('A1:C1', "Price Matrix (Pivot)", self.fmt_title)
ws.set_row(0, 25)
if not price_history or not stocks:
ws.write(3, 0, "No data available")
return
# Get unique dates (limit to 365)
dates = sorted(set(p['scrape_date'] for p in price_history if p.get('scrape_date')),
reverse=True)[:365]
if not dates:
ws.write(3, 0, "No date data available")
return
# Limit to 100 stocks
stock_list = stocks[:100]
# Build price lookup
price_lookup = {}
for p in price_history:
key = (p['stock_id'], p['scrape_date'])
price_lookup[key] = p.get('price')
# Header row with tickers
row = 3
ws.write(row, 0, "Date", self.fmt_header)
for col, stock in enumerate(stock_list, 1):
ws.write(row, col, stock['ticker'], self.fmt_header)
# Data rows
for row_idx, date in enumerate(dates):
ws.write(row + 1 + row_idx, 0, date)
for col_idx, stock in enumerate(stock_list, 1):
price = price_lookup.get((stock['id'], date))
if price:
ws.write(row + 1 + row_idx, col_idx, price, self.fmt_num)
ws.freeze_panes(4, 1)
ws.set_column(0, 0, 12)
ws.set_column(1, 100, 10)
def generate(self):
"""Generate all history sheets."""
print("Loading data...")
stocks = self._get_stocks()
price_history = self._get_price_history()
ratios_history = self._get_ratios_history()
sentiment_history = self._get_sentiment_history()
scrape_runs = self._get_scrape_runs()
print("Creating Summary sheet...")
self.create_summary_sheet(stocks, price_history, ratios_history,
sentiment_history, scrape_runs)
print("Creating Price History sheet...")
self.create_price_history_sheet(price_history)
print("Creating Ratios History sheet...")
self.create_ratios_history_sheet(ratios_history)
print("Creating Sentiment History sheet...")
self.create_sentiment_history_sheet(sentiment_history)
if stocks and price_history:
print("Creating Price Pivot sheet...")
self.create_price_pivot_sheet(stocks, price_history)
def main():
parser = argparse.ArgumentParser(
description="Export historical stock data to professional Excel"
)
parser.add_argument('--db', required=True, help='Path to SQLite database')
parser.add_argument('--output', '-o', required=True, help='Output Excel file path')
parser.add_argument('--start-date', help='Start date filter (YYYY-MM-DD)')
parser.add_argument('--end-date', help='End date filter (YYYY-MM-DD)')
parser.add_argument('--stock', help='Filter by stock ID')
args = parser.parse_args()
if not Path(args.db).exists():
print(f"Error: Database not found: {args.db}")
return 1
conn = sqlite3.connect(args.db)
conn.row_factory = sqlite3.Row
workbook = xlsxwriter.Workbook(args.output, {
'constant_memory': False,
'strings_to_urls': True,
})
exporter = HistoryExporter(
workbook, conn,
start_date=args.start_date,
end_date=args.end_date,
stock_id=args.stock
)
exporter.generate()
workbook.close()
conn.close()
print(f"\nHistory exported to: {args.output}")
return 0
if __name__ == "__main__":
exit(main())

81
scripts/export_simple.py Normal file
View file

@ -0,0 +1,81 @@
#!/usr/bin/env python3
"""Simple table export from SQLite to JSON/CSV/XLSX."""
import argparse
import json
import sqlite3
from pathlib import Path
import pandas as pd
DEFAULT_TABLES = [
"stocks",
"price_history",
"ratios_history",
"news",
"sentiment_history",
"scrape_runs",
"scrape_progress",
]
def load_table(conn: sqlite3.Connection, table: str) -> pd.DataFrame:
return pd.read_sql_query(f"SELECT * FROM {table}", conn)
def export_json(conn: sqlite3.Connection, outdir: Path, tables: list[str]) -> int:
outdir.mkdir(parents=True, exist_ok=True)
for table in tables:
df = load_table(conn, table)
data = json.loads(df.to_json(orient="records", date_format="iso"))
(outdir / f"{table}.json").write_text(json.dumps(data, indent=2), encoding="utf-8")
return 0
def export_csv(conn: sqlite3.Connection, outdir: Path, tables: list[str]) -> int:
outdir.mkdir(parents=True, exist_ok=True)
for table in tables:
df = load_table(conn, table)
df.to_csv(outdir / f"{table}.csv", index=False)
return 0
def export_xlsx(conn: sqlite3.Connection, outfile: Path, tables: list[str]) -> int:
outfile.parent.mkdir(parents=True, exist_ok=True)
with pd.ExcelWriter(outfile, engine="openpyxl") as writer:
for table in tables:
df = load_table(conn, table)
sheet = table[:31] if table else "sheet"
df.to_excel(writer, sheet_name=sheet, index=False)
return 0
def main() -> int:
parser = argparse.ArgumentParser(description="Simple SQLite table exporter")
parser.add_argument("--db", required=True, help="SQLite database path")
parser.add_argument("--format", required=True, choices=["json", "csv", "xlsx"], help="Export format")
parser.add_argument("--output", "-o", required=True, help="Output path (dir for json/csv, file for xlsx)")
parser.add_argument("--tables", help="Comma-separated tables (default: common tables)")
args = parser.parse_args()
db_path = Path(args.db)
if not db_path.exists():
print(f"Error: Database not found: {db_path}")
return 1
tables = [t.strip() for t in args.tables.split(",")] if args.tables else DEFAULT_TABLES
tables = [t for t in tables if t]
conn = sqlite3.connect(db_path)
try:
if args.format == "json":
return export_json(conn, Path(args.output), tables)
if args.format == "csv":
return export_csv(conn, Path(args.output), tables)
return export_xlsx(conn, Path(args.output), tables)
finally:
conn.close()
if __name__ == "__main__":
raise SystemExit(main())

65
scripts/release_bundle.sh Executable file
View file

@ -0,0 +1,65 @@
#!/usr/bin/env bash
set -euo pipefail
VERSION="${1:-}"
if [[ -z "$VERSION" ]]; then
if git describe --tags --always >/dev/null 2>&1; then
VERSION="$(git describe --tags --always)"
else
VERSION="0.0.0-$(date +%Y%m%d%H%M%S)"
fi
fi
APP_NAME="rubick"
OS="$(uname -s | tr '[:upper:]' '[:lower:]')"
ARCH="$(uname -m)"
case "$ARCH" in
x86_64) ARCH="amd64" ;;
aarch64|arm64) ARCH="arm64" ;;
esac
DIST_DIR="dist"
PKG_BASENAME="${APP_NAME}_${VERSION}_${OS}_${ARCH}"
PKG_DIR="${DIST_DIR}/${PKG_BASENAME}"
rm -rf "$PKG_DIR"
mkdir -p "$PKG_DIR/bin" "$PKG_DIR/scripts"
# Build binary
GOOS="$OS" GOARCH="$ARCH" go build -o "$PKG_DIR/bin/$APP_NAME" ./cmd/rubick
# Runtime Python assets
cp extractor.py "$PKG_DIR/"
cp scripts/export_dashboard.py scripts/export_history.py scripts/export_simple.py "$PKG_DIR/scripts/"
cp pyproject.toml uv.lock .env.example README.md "$PKG_DIR/"
cat > "$PKG_DIR/INSTALL.md" <<'DOC'
# Rubick Bundle Install
1. Ensure Python 3.12+ and uv are installed.
2. In this bundle directory, run:
```bash
uv sync --frozen
```
3. Run the binary:
```bash
./bin/rubick --help
```
4. For live news queries, create `.env` with `BRAVE_API_KEY`.
DOC
# Checksums
( cd "$PKG_DIR" && shasum -a 256 bin/$APP_NAME extractor.py scripts/*.py pyproject.toml uv.lock > SHA256SUMS )
# Archives
( cd "$DIST_DIR" && tar -czf "${PKG_BASENAME}.tar.gz" "$PKG_BASENAME" )
( cd "$DIST_DIR" && zip -qr "${PKG_BASENAME}.zip" "$PKG_BASENAME" )
echo "release_bundle=$PKG_DIR"
echo "archive_tar=${DIST_DIR}/${PKG_BASENAME}.tar.gz"
echo "archive_zip=${DIST_DIR}/${PKG_BASENAME}.zip"