mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-08-07 01:33:52 +00:00
feat(rubick): rename project and add production release bundle
This commit is contained in:
Muhammad Firas
commit
fab6cf26eb
33 changed files with 11554 additions and 0 deletions
129
internal/cli/brave.go
Normal file
129
internal/cli/brave.go
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/enetx/g"
|
||||
"github.com/enetx/surf"
|
||||
)
|
||||
|
||||
// BraveResult represents a single news result from Brave Search API
|
||||
type BraveResult struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Description string `json:"description"`
|
||||
PageAge string `json:"page_age"`
|
||||
}
|
||||
|
||||
// BraveNewsResponse represents the Brave News Search API response
|
||||
type BraveNewsResponse struct {
|
||||
Results []struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Description string `json:"description"`
|
||||
Age string `json:"age"`
|
||||
} `json:"results"`
|
||||
}
|
||||
|
||||
// SearchConfig holds search parameters
|
||||
type SearchConfig struct {
|
||||
Query string
|
||||
From time.Time
|
||||
To time.Time
|
||||
Count int
|
||||
}
|
||||
|
||||
// SearchBrave queries the Brave News Search API
|
||||
// Uses dedicated News endpoint: GET /res/v1/news/search
|
||||
func SearchBrave(client *surf.Client, config SearchConfig) ([]BraveResult, error) {
|
||||
apiKey := os.Getenv("BRAVE_API_KEY")
|
||||
if apiKey == "" {
|
||||
return nil, fmt.Errorf("BRAVE_API_KEY environment variable not set")
|
||||
}
|
||||
|
||||
// Build query parameters
|
||||
params := url.Values{}
|
||||
params.Set("q", config.Query)
|
||||
params.Set("count", fmt.Sprintf("%d", config.Count))
|
||||
params.Set("freshness", fmt.Sprintf("%sto%s",
|
||||
config.From.Format("2006-01-02"),
|
||||
config.To.Format("2006-01-02"),
|
||||
))
|
||||
|
||||
apiURL := fmt.Sprintf("https://api.search.brave.com/res/v1/news/search?%s", params.Encode())
|
||||
|
||||
// Use plain client for API calls (no Chrome impersonation which overrides headers)
|
||||
apiClient := surf.NewClient()
|
||||
defer apiClient.CloseIdleConnections()
|
||||
|
||||
resp := apiClient.Get(g.String(apiURL)).
|
||||
SetHeaders("Accept", "application/json").
|
||||
SetHeaders("X-Subscription-Token", apiKey).
|
||||
// Jakarta, Indonesia location headers
|
||||
SetHeaders("X-Loc-Lat", "-6.2088").
|
||||
SetHeaders("X-Loc-Long", "106.8456").
|
||||
SetHeaders("X-Loc-Timezone", "Asia/Jakarta").
|
||||
SetHeaders("X-Loc-Country", "ID").
|
||||
Do()
|
||||
|
||||
if resp.IsErr() {
|
||||
return nil, fmt.Errorf("brave API request failed: %w", resp.Err())
|
||||
}
|
||||
|
||||
r := resp.Ok()
|
||||
if r.StatusCode != 200 {
|
||||
body := r.Body.String().Ok().Std()
|
||||
return nil, fmt.Errorf("brave API returned status %d: %s", r.StatusCode, body)
|
||||
}
|
||||
|
||||
body := r.Body.String().Ok().Std()
|
||||
|
||||
var braveResp BraveNewsResponse
|
||||
if err := json.Unmarshal([]byte(body), &braveResp); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse brave API response: %w", err)
|
||||
}
|
||||
|
||||
if len(braveResp.Results) == 0 {
|
||||
return []BraveResult{}, nil
|
||||
}
|
||||
|
||||
results := make([]BraveResult, len(braveResp.Results))
|
||||
for i, r := range braveResp.Results {
|
||||
results[i] = BraveResult{
|
||||
Title: r.Title,
|
||||
URL: r.URL,
|
||||
Description: r.Description,
|
||||
PageAge: r.Age,
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// BuildStockQuery creates a boolean query for Indonesian stock news
|
||||
// Example: BuildStockQuery("MINA", "MINA Tbk") returns:
|
||||
// ("MINA" OR "MINA Tbk") AND (saham OR emiten OR "Bursa Efek Indonesia" OR BEI OR IDX)
|
||||
func BuildStockQuery(stockTerms ...string) string {
|
||||
if len(stockTerms) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Build stock terms part
|
||||
stockPart := "("
|
||||
for i, term := range stockTerms {
|
||||
if i > 0 {
|
||||
stockPart += " OR "
|
||||
}
|
||||
stockPart += fmt.Sprintf(`"%s"`, term)
|
||||
}
|
||||
stockPart += ")"
|
||||
|
||||
// Indonesian stock market keywords
|
||||
marketKeywords := `(saham OR emiten OR "Bursa Efek Indonesia" OR BEI OR IDX)`
|
||||
|
||||
return stockPart + " AND " + marketKeywords
|
||||
}
|
||||
282
internal/cli/extractor.go
Normal file
282
internal/cli/extractor.go
Normal file
|
|
@ -0,0 +1,282 @@
|
|||
package cli
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ExtractRequest is the request sent to Python extractor
|
||||
type ExtractRequest struct {
|
||||
URL string `json:"url"`
|
||||
HTML string `json:"html"`
|
||||
}
|
||||
|
||||
// ExtractResponse is the response from Python extractor
|
||||
type ExtractResponse struct {
|
||||
Text string `json:"text"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Extractor manages the Python extraction process and socket communication
|
||||
type Extractor struct {
|
||||
socketPath string
|
||||
cmd *exec.Cmd
|
||||
mu sync.Mutex
|
||||
connPool chan net.Conn
|
||||
poolSize int
|
||||
closed bool
|
||||
}
|
||||
|
||||
// NewExtractor creates and starts the Python extractor process
|
||||
func NewExtractor(poolSize int) (*Extractor, error) {
|
||||
if poolSize < 1 {
|
||||
poolSize = 1
|
||||
}
|
||||
|
||||
socketPath := filepath.Join(os.TempDir(), fmt.Sprintf("stock-news-extractor-%d-%d.sock", os.Getpid(), time.Now().UnixNano()))
|
||||
|
||||
e := &Extractor{
|
||||
socketPath: socketPath,
|
||||
poolSize: poolSize,
|
||||
connPool: make(chan net.Conn, poolSize),
|
||||
}
|
||||
|
||||
if err := e.start(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// start launches the Python extractor process
|
||||
func (e *Extractor) start() error {
|
||||
// Remove existing socket file if present
|
||||
os.Remove(e.socketPath)
|
||||
|
||||
e.cmd = exec.Command("uv", "run", "python", "extractor.py", "--socket", e.socketPath)
|
||||
e.cmd.Stderr = os.Stderr
|
||||
|
||||
// Create a new process group so we can kill all child processes
|
||||
e.cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
|
||||
stdout, err := e.cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create stdout pipe: %w", err)
|
||||
}
|
||||
|
||||
if err := e.cmd.Start(); err != nil {
|
||||
return fmt.Errorf("failed to start python extractor: %w", err)
|
||||
}
|
||||
|
||||
// Wait for ready signal from Python
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
ready := make(chan bool, 1)
|
||||
go func() {
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.HasPrefix(line, "READY:") {
|
||||
ready <- true
|
||||
return
|
||||
}
|
||||
}
|
||||
ready <- false
|
||||
}()
|
||||
|
||||
select {
|
||||
case ok := <-ready:
|
||||
if !ok {
|
||||
e.killProcessGroup()
|
||||
return fmt.Errorf("python extractor failed to start")
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
e.killProcessGroup()
|
||||
return fmt.Errorf("timeout waiting for python extractor to start")
|
||||
}
|
||||
|
||||
// Initialize connection pool
|
||||
for i := 0; i < e.poolSize; i++ {
|
||||
conn, err := net.Dial("unix", e.socketPath)
|
||||
if err != nil {
|
||||
e.Close()
|
||||
return fmt.Errorf("failed to connect to extractor: %w", err)
|
||||
}
|
||||
e.connPool <- conn
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// killProcessGroup kills the entire process group
|
||||
func (e *Extractor) killProcessGroup() {
|
||||
if e.cmd != nil && e.cmd.Process != nil {
|
||||
// Kill the entire process group (negative PID)
|
||||
pgid, err := syscall.Getpgid(e.cmd.Process.Pid)
|
||||
if err == nil {
|
||||
syscall.Kill(-pgid, syscall.SIGTERM)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
syscall.Kill(-pgid, syscall.SIGKILL)
|
||||
}
|
||||
e.cmd.Process.Kill()
|
||||
e.cmd.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
// Extract sends HTML to Python and returns extracted text
|
||||
func (e *Extractor) Extract(ctx context.Context, url, html string) (*ExtractResponse, error) {
|
||||
// Get connection from pool
|
||||
var conn net.Conn
|
||||
select {
|
||||
case conn = <-e.connPool:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
healthy := true
|
||||
// Return connection to pool when done
|
||||
defer func() {
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
if !e.closed && healthy {
|
||||
e.connPool <- conn
|
||||
return
|
||||
}
|
||||
_ = conn.Close()
|
||||
if e.closed {
|
||||
return
|
||||
}
|
||||
replacement, err := net.Dial("unix", e.socketPath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case e.connPool <- replacement:
|
||||
default:
|
||||
_ = replacement.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = conn.SetDeadline(dl)
|
||||
} else {
|
||||
_ = conn.SetDeadline(time.Now().Add(60 * time.Second))
|
||||
}
|
||||
defer conn.SetDeadline(time.Time{})
|
||||
|
||||
req := ExtractRequest{URL: url, HTML: html}
|
||||
if err := writeMessage(conn, req); err != nil {
|
||||
healthy = false
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
|
||||
var resp ExtractResponse
|
||||
if err := readMessage(conn, &resp); err != nil {
|
||||
healthy = false
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
// Close shuts down the Python extractor
|
||||
func (e *Extractor) Close() error {
|
||||
e.mu.Lock()
|
||||
if e.closed {
|
||||
e.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
e.closed = true
|
||||
e.mu.Unlock()
|
||||
|
||||
// Drain and close all connections in pool
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
for i := 0; i < e.poolSize; i++ {
|
||||
select {
|
||||
case conn := <-e.connPool:
|
||||
conn.Close()
|
||||
case <-time.After(time.Second):
|
||||
// Timeout waiting for connection
|
||||
}
|
||||
}
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
// Timeout waiting for pool drain
|
||||
}
|
||||
|
||||
// Kill the process group
|
||||
e.killProcessGroup()
|
||||
|
||||
// Clean up socket file
|
||||
_ = os.Remove(e.socketPath)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeMessage writes a length-prefixed JSON message
|
||||
func writeMessage(conn net.Conn, msg interface{}) error {
|
||||
data, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write 4-byte length prefix (big-endian)
|
||||
header := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(header, uint32(len(data)))
|
||||
|
||||
if err := writeAll(conn, header); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeAll(conn, data); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// readMessage reads a length-prefixed JSON message
|
||||
func readMessage(conn net.Conn, v interface{}) error {
|
||||
// Read 4-byte length prefix
|
||||
header := make([]byte, 4)
|
||||
if _, err := io.ReadFull(conn, header); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
length := binary.BigEndian.Uint32(header)
|
||||
|
||||
// Read JSON payload
|
||||
data := make([]byte, length)
|
||||
if _, err := io.ReadFull(conn, data); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return json.Unmarshal(data, v)
|
||||
}
|
||||
|
||||
func writeAll(conn net.Conn, p []byte) error {
|
||||
for len(p) > 0 {
|
||||
n, err := conn.Write(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p = p[n:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
143
internal/cli/main.go
Normal file
143
internal/cli/main.go
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
func Run(args []string) int {
|
||||
if len(args) == 0 {
|
||||
printRootUsage()
|
||||
return 1
|
||||
}
|
||||
|
||||
if args[0] == "-h" || args[0] == "--help" || args[0] == "help" {
|
||||
printRootUsage()
|
||||
return 0
|
||||
}
|
||||
|
||||
switch args[0] {
|
||||
case "msn":
|
||||
return runMSNCommand(args[1:])
|
||||
case "news":
|
||||
return runNewsCommand(args[1:])
|
||||
case "export":
|
||||
return runExportCommand(args[1:])
|
||||
case "extractor":
|
||||
return runExtractorCommand(args[1:])
|
||||
default:
|
||||
printRootUsage()
|
||||
fmt.Fprintf(os.Stderr, "\nerror: unknown command: %s\n", args[0])
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
func printRootUsage() {
|
||||
fmt.Fprintf(os.Stderr, `Rubick - Unified Market Intelligence CLI
|
||||
|
||||
Usage:
|
||||
rubick <command> [options]
|
||||
|
||||
Commands:
|
||||
msn MSN finance workflows (screener, fetch, fetch-all, lookup)
|
||||
news Brave news search + Python text extraction
|
||||
export Python export tools (dashboard/history/simple)
|
||||
extractor Run raw Python extractor server (advanced)
|
||||
|
||||
Examples:
|
||||
# News mode with explicit command
|
||||
rubick news "BBCA,Bank Central Asia" --stock --count 20
|
||||
|
||||
# MSN screener
|
||||
rubick msn screener --region id --filter top-performers --limit 20 -o output/screener.json
|
||||
|
||||
# MSN fetch-all to SQLite
|
||||
rubick msn fetch-all --index idx30 --db output/stocks.db --concurrency 3
|
||||
|
||||
# Export dashboard from SQLite
|
||||
rubick export dashboard --db output/stocks.db --output output/dashboard.xlsx
|
||||
|
||||
# Export history workbook from SQLite
|
||||
rubick export history --db output/stocks.db --output output/history.xlsx
|
||||
|
||||
# Export simple tables (json/csv/xlsx) from SQLite
|
||||
rubick export simple --db output/stocks.db --format csv --output output/simple_csv
|
||||
|
||||
# Run extractor server directly
|
||||
rubick extractor --socket /tmp/extractor.sock
|
||||
`)
|
||||
}
|
||||
|
||||
func runExportCommand(args []string) int {
|
||||
if len(args) == 0 {
|
||||
fmt.Fprintf(os.Stderr, `Usage: rubick export <dashboard|history|simple> [script options]
|
||||
|
||||
Examples:
|
||||
rubick export dashboard --db output/stocks.db --output output/dashboard.xlsx
|
||||
rubick export history --db output/stocks.db --output output/history.xlsx
|
||||
rubick export simple --db output/stocks.db --format csv --output output/csv/
|
||||
`)
|
||||
return 1
|
||||
}
|
||||
|
||||
if args[0] == "-h" || args[0] == "--help" || args[0] == "help" {
|
||||
fmt.Fprintf(os.Stderr, `Usage: rubick export <dashboard|history|simple> [script options]
|
||||
|
||||
Examples:
|
||||
rubick export dashboard --db output/stocks.db --output output/dashboard.xlsx
|
||||
rubick export history --db output/stocks.db --output output/history.xlsx
|
||||
rubick export simple --db output/stocks.db --format csv --output output/csv/
|
||||
`)
|
||||
return 0
|
||||
}
|
||||
|
||||
script := ""
|
||||
switch args[0] {
|
||||
case "dashboard":
|
||||
script = "scripts/export_dashboard.py"
|
||||
case "history":
|
||||
script = "scripts/export_history.py"
|
||||
case "simple":
|
||||
script = "scripts/export_simple.py"
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "Unknown export target: %s\n", args[0])
|
||||
return 1
|
||||
}
|
||||
|
||||
cmdArgs := append([]string{"run", "python", script}, args[1:]...)
|
||||
return runPassthrough("uv", cmdArgs)
|
||||
}
|
||||
|
||||
func runExtractorCommand(args []string) int {
|
||||
if len(args) == 0 {
|
||||
fmt.Fprintf(os.Stderr, "Usage: rubick extractor --socket /tmp/extractor.sock\n")
|
||||
return 1
|
||||
}
|
||||
|
||||
if args[0] == "-h" || args[0] == "--help" || args[0] == "help" {
|
||||
fmt.Fprintf(os.Stderr, "Usage: rubick extractor --socket /tmp/extractor.sock\n")
|
||||
return 0
|
||||
}
|
||||
|
||||
cmdArgs := append([]string{"run", "python", "extractor.py"}, args...)
|
||||
return runPassthrough("uv", cmdArgs)
|
||||
}
|
||||
|
||||
func runPassthrough(bin string, args []string) int {
|
||||
cmd := exec.Command(bin, args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Stdin = os.Stdin
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
var exitErr *exec.ExitError
|
||||
if errors.As(err, &exitErr) {
|
||||
return exitErr.ExitCode()
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "failed to run %s: %v\n", bin, err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
904
internal/cli/msn_cli.go
Normal file
904
internal/cli/msn_cli.go
Normal file
|
|
@ -0,0 +1,904 @@
|
|||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"rubick/msn"
|
||||
)
|
||||
|
||||
func runMSNCommand(args []string) int {
|
||||
if len(args) == 0 {
|
||||
printMSNUsage()
|
||||
return 1
|
||||
}
|
||||
|
||||
if args[0] == "-h" || args[0] == "--help" || args[0] == "help" {
|
||||
printMSNUsage()
|
||||
return 0
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
defer signal.Stop(sigChan)
|
||||
go func() {
|
||||
<-sigChan
|
||||
log.Println("Shutting down...")
|
||||
cancel()
|
||||
}()
|
||||
|
||||
if err := executeMSNCommand(ctx, args); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func executeMSNCommand(ctx context.Context, args []string) error {
|
||||
subcommand := args[0]
|
||||
subArgs := args[1:]
|
||||
|
||||
switch subcommand {
|
||||
case "screener":
|
||||
return runScreener(ctx, subArgs)
|
||||
case "fetch":
|
||||
return runFetch(ctx, subArgs)
|
||||
case "fetch-all":
|
||||
return runFetchAll(ctx, subArgs)
|
||||
case "lookup":
|
||||
return runLookup(subArgs)
|
||||
default:
|
||||
printMSNUsage()
|
||||
return fmt.Errorf("unknown msn subcommand: %s", subcommand)
|
||||
}
|
||||
}
|
||||
|
||||
func printMSNUsage() {
|
||||
fmt.Fprintf(os.Stderr, `MSN Stock Scraper - Fetch Indonesian stock data from MSN Finance
|
||||
|
||||
Usage:
|
||||
rubick msn <command> [options]
|
||||
|
||||
Commands:
|
||||
screener Run stock screener to find stocks by criteria
|
||||
fetch Fetch comprehensive data for specific stocks
|
||||
fetch-all Fetch ALL Indonesian stocks to SQLite database
|
||||
lookup Look up MSN ID for ticker symbols
|
||||
|
||||
Screener:
|
||||
rubick msn screener --region id --filter top-performers --limit 20 -o stocks.json
|
||||
|
||||
Fetch:
|
||||
rubick msn fetch --tickers BBCA,BBRI,TLKM -o bank_stocks.json
|
||||
rubick msn fetch --input stocks.json -o full_data.json
|
||||
|
||||
Fetch-All:
|
||||
rubick msn fetch-all --index idx30 --db output/stocks.db --concurrency 3
|
||||
|
||||
Lookup:
|
||||
rubick msn lookup BBCA BBRI TLKM
|
||||
`)
|
||||
}
|
||||
|
||||
// Screener
|
||||
|
||||
type ScreenerCLIConfig struct {
|
||||
Region string
|
||||
Filter string
|
||||
Limit int
|
||||
Output string
|
||||
}
|
||||
|
||||
func parseScreenerArgs(args []string) (ScreenerCLIConfig, error) {
|
||||
cfg := ScreenerCLIConfig{
|
||||
Region: "id",
|
||||
Filter: "large-cap",
|
||||
Limit: 50,
|
||||
Output: fmt.Sprintf("screener_%s.json", time.Now().Format("20060102")),
|
||||
}
|
||||
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch args[i] {
|
||||
case "--region":
|
||||
v, n, err := requireValue(args, i, "--region")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
cfg.Region = v
|
||||
i = n
|
||||
case "--filter":
|
||||
v, n, err := requireValue(args, i, "--filter")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
cfg.Filter = v
|
||||
i = n
|
||||
case "--limit":
|
||||
v, n, err := requireValue(args, i, "--limit")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
limit, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return cfg, fmt.Errorf("invalid --limit value: %w", err)
|
||||
}
|
||||
cfg.Limit = limit
|
||||
i = n
|
||||
case "--output", "-o":
|
||||
v, n, err := requireValue(args, i, "--output")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
cfg.Output = v
|
||||
i = n
|
||||
default:
|
||||
return cfg, fmt.Errorf("unknown option: %s", args[i])
|
||||
}
|
||||
}
|
||||
if cfg.Limit < 1 {
|
||||
return cfg, fmt.Errorf("--limit must be >= 1")
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func runScreener(ctx context.Context, args []string) error {
|
||||
if wantsHelp(args) {
|
||||
printMSNScreenerUsage()
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg, err := parseScreenerArgs(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
filter, err := msn.ParseScreenerFilter(cfg.Filter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid filter: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("Running screener: region=%s filter=%s limit=%d", cfg.Region, cfg.Filter, cfg.Limit)
|
||||
client := msn.NewMSNClient()
|
||||
defer client.Close()
|
||||
|
||||
result, err := client.RunScreener(msn.ScreenerConfig{Region: cfg.Region, Filter: filter, Limit: cfg.Limit})
|
||||
if err != nil {
|
||||
return fmt.Errorf("screener failed: %w", err)
|
||||
}
|
||||
|
||||
output := msn.ScreenerOutput{
|
||||
Filter: cfg.Filter,
|
||||
Region: cfg.Region,
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
Total: result.Total,
|
||||
Stocks: result.Value,
|
||||
}
|
||||
if err := saveJSON(output, cfg.Output); err != nil {
|
||||
return fmt.Errorf("failed to save output: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("Output saved to %s", cfg.Output)
|
||||
fmt.Println("\nTop 10 results:")
|
||||
for i, stock := range result.Value {
|
||||
if i >= 10 {
|
||||
break
|
||||
}
|
||||
fmt.Printf(" %s (%s): %.2f (%.2f%%)\n", stock.Symbol, stock.ID, stock.Price, stock.PriceChangePct)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Fetch
|
||||
|
||||
type FetchCLIConfig struct {
|
||||
Input string
|
||||
IDs []string
|
||||
Tickers []string
|
||||
Concurrency int
|
||||
Output string
|
||||
}
|
||||
|
||||
func parseFetchArgs(args []string) (FetchCLIConfig, error) {
|
||||
cfg := FetchCLIConfig{Concurrency: 5, Output: fmt.Sprintf("stocks_%s.json", time.Now().Format("20060102"))}
|
||||
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch args[i] {
|
||||
case "--input":
|
||||
v, n, err := requireValue(args, i, "--input")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
cfg.Input = v
|
||||
i = n
|
||||
case "--ids":
|
||||
v, n, err := requireValue(args, i, "--ids")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
cfg.IDs = appendCSV(cfg.IDs, v, false)
|
||||
i = n
|
||||
case "--tickers":
|
||||
v, n, err := requireValue(args, i, "--tickers")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
cfg.Tickers = appendCSV(cfg.Tickers, v, true)
|
||||
i = n
|
||||
case "--concurrency":
|
||||
v, n, err := requireValue(args, i, "--concurrency")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
conc, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return cfg, fmt.Errorf("invalid --concurrency value: %w", err)
|
||||
}
|
||||
cfg.Concurrency = conc
|
||||
i = n
|
||||
case "--output", "-o":
|
||||
v, n, err := requireValue(args, i, "--output")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
cfg.Output = v
|
||||
i = n
|
||||
default:
|
||||
return cfg, fmt.Errorf("unknown option: %s", args[i])
|
||||
}
|
||||
}
|
||||
if cfg.Concurrency < 1 {
|
||||
return cfg, fmt.Errorf("--concurrency must be >= 1")
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func runFetch(ctx context.Context, args []string) error {
|
||||
if wantsHelp(args) {
|
||||
printMSNFetchUsage()
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg, err := parseFetchArgs(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ids := make([]string, 0)
|
||||
if cfg.Input != "" {
|
||||
inputIDs, err := readScreenerOutput(cfg.Input)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read input file: %w", err)
|
||||
}
|
||||
ids = append(ids, inputIDs...)
|
||||
}
|
||||
ids = append(ids, cfg.IDs...)
|
||||
|
||||
for _, ticker := range cfg.Tickers {
|
||||
id := msn.GetIDXStockID(ticker)
|
||||
if id == "" {
|
||||
log.Printf("Warning: Unknown ticker '%s', skipping", ticker)
|
||||
continue
|
||||
}
|
||||
log.Printf("Resolved %s -> %s", ticker, id)
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
if len(ids) == 0 {
|
||||
return fmt.Errorf("no stock IDs provided. use --input, --ids, or --tickers")
|
||||
}
|
||||
|
||||
ids = dedupe(ids)
|
||||
log.Printf("Fetching data for %d stocks with concurrency %d", len(ids), cfg.Concurrency)
|
||||
|
||||
fetcher := msn.NewStockFetcher()
|
||||
defer fetcher.Close()
|
||||
|
||||
stocks := fetcher.FetchStocks(ctx, ids, cfg.Concurrency)
|
||||
output := msn.FetchOutput{GeneratedAt: time.Now().UTC().Format(time.RFC3339), Total: len(stocks), Stocks: stocks}
|
||||
if err := saveJSON(output, cfg.Output); err != nil {
|
||||
return fmt.Errorf("failed to save output: %w", err)
|
||||
}
|
||||
|
||||
successCount := 0
|
||||
for _, stock := range stocks {
|
||||
apiSuccess := 0
|
||||
for _, status := range stock.FetchStatus {
|
||||
if status == "ok" {
|
||||
apiSuccess++
|
||||
}
|
||||
}
|
||||
if apiSuccess > 0 {
|
||||
successCount++
|
||||
}
|
||||
}
|
||||
log.Printf("Output saved to %s", cfg.Output)
|
||||
log.Printf("Successfully fetched %d/%d stocks", successCount, len(stocks))
|
||||
return nil
|
||||
}
|
||||
|
||||
func readScreenerOutput(filename string) ([]string, error) {
|
||||
data, err := os.ReadFile(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var output msn.ScreenerOutput
|
||||
if err := json.Unmarshal(data, &output); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ids := make([]string, len(output.Stocks))
|
||||
for i, stock := range output.Stocks {
|
||||
ids[i] = stock.ID
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func saveJSON(data any, filename string) error {
|
||||
jsonData, err := json.MarshalIndent(data, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir := filepath.Dir(filename); dir != "." {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return os.WriteFile(filename, jsonData, 0o644)
|
||||
}
|
||||
|
||||
func runLookup(args []string) error {
|
||||
if wantsHelp(args) {
|
||||
printMSNLookupUsage()
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("usage: rubick msn lookup <ticker1> [ticker2] ...")
|
||||
}
|
||||
|
||||
fmt.Printf("%-8s %-10s %s\n", "Ticker", "MSN ID", "Company Name")
|
||||
fmt.Println(strings.Repeat("-", 50))
|
||||
|
||||
found := 0
|
||||
for _, ticker := range args {
|
||||
ticker = strings.ToUpper(strings.TrimSpace(ticker))
|
||||
stock, ok := msn.GetIDXStock(ticker)
|
||||
if ok {
|
||||
fmt.Printf("%-8s %-10s %s\n", ticker, stock.ID, stock.Name)
|
||||
found++
|
||||
} else {
|
||||
fmt.Printf("%-8s %-10s %s\n", ticker, "-", "(not found)")
|
||||
}
|
||||
}
|
||||
fmt.Println(strings.Repeat("-", 50))
|
||||
fmt.Printf("Found %d/%d tickers\n", found, len(args))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Fetch-all
|
||||
|
||||
type FetchAllConfig struct {
|
||||
DB string
|
||||
Index string
|
||||
Proxy string
|
||||
Concurrency int
|
||||
RPS float64
|
||||
MinDelayMs int
|
||||
MaxDelayMs int
|
||||
Retry int
|
||||
Limit int
|
||||
Resume bool
|
||||
}
|
||||
|
||||
func parseFetchAllArgs(args []string) (FetchAllConfig, error) {
|
||||
cfg := FetchAllConfig{
|
||||
DB: "output/stocks.db",
|
||||
Index: "all",
|
||||
Concurrency: 5,
|
||||
RPS: 25,
|
||||
MinDelayMs: 100,
|
||||
MaxDelayMs: 500,
|
||||
Retry: 2,
|
||||
}
|
||||
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch args[i] {
|
||||
case "--db":
|
||||
v, n, err := requireValue(args, i, "--db")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
cfg.DB = v
|
||||
i = n
|
||||
case "--index":
|
||||
v, n, err := requireValue(args, i, "--index")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
cfg.Index = strings.ToLower(v)
|
||||
i = n
|
||||
case "--proxy":
|
||||
v, n, err := requireValue(args, i, "--proxy")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
cfg.Proxy = v
|
||||
i = n
|
||||
case "--concurrency":
|
||||
v, n, err := requireValue(args, i, "--concurrency")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
nval, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return cfg, fmt.Errorf("invalid --concurrency value: %w", err)
|
||||
}
|
||||
cfg.Concurrency = nval
|
||||
i = n
|
||||
case "--rps":
|
||||
v, n, err := requireValue(args, i, "--rps")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
fval, err := strconv.ParseFloat(v, 64)
|
||||
if err != nil {
|
||||
return cfg, fmt.Errorf("invalid --rps value: %w", err)
|
||||
}
|
||||
cfg.RPS = fval
|
||||
i = n
|
||||
case "--delay":
|
||||
v, n, err := requireValue(args, i, "--delay")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
minDelay, maxDelay, err := parseDelay(v)
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
cfg.MinDelayMs, cfg.MaxDelayMs = minDelay, maxDelay
|
||||
i = n
|
||||
case "--retry":
|
||||
v, n, err := requireValue(args, i, "--retry")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
nval, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return cfg, fmt.Errorf("invalid --retry value: %w", err)
|
||||
}
|
||||
cfg.Retry = nval
|
||||
i = n
|
||||
case "--limit":
|
||||
v, n, err := requireValue(args, i, "--limit")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
nval, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return cfg, fmt.Errorf("invalid --limit value: %w", err)
|
||||
}
|
||||
cfg.Limit = nval
|
||||
i = n
|
||||
case "--resume":
|
||||
cfg.Resume = true
|
||||
default:
|
||||
return cfg, fmt.Errorf("unknown option: %s", args[i])
|
||||
}
|
||||
}
|
||||
if cfg.Concurrency < 1 {
|
||||
return cfg, fmt.Errorf("--concurrency must be >= 1")
|
||||
}
|
||||
if cfg.RPS <= 0 {
|
||||
return cfg, fmt.Errorf("--rps must be > 0")
|
||||
}
|
||||
if cfg.MinDelayMs < 0 || cfg.MaxDelayMs < 0 || cfg.MinDelayMs > cfg.MaxDelayMs {
|
||||
return cfg, fmt.Errorf("--delay must satisfy 0 <= min <= max")
|
||||
}
|
||||
if cfg.Retry < 0 {
|
||||
return cfg, fmt.Errorf("--retry must be >= 0")
|
||||
}
|
||||
if cfg.Limit < 0 {
|
||||
return cfg, fmt.Errorf("--limit must be >= 0")
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func runFetchAll(ctx context.Context, args []string) error {
|
||||
if wantsHelp(args) {
|
||||
printMSNFetchAllUsage()
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg, err := parseFetchAllArgs(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Fetch-All Configuration:")
|
||||
log.Printf(" Database: %s", cfg.DB)
|
||||
log.Printf(" Index: %s", cfg.Index)
|
||||
log.Printf(" Concurrency: %d workers", cfg.Concurrency)
|
||||
log.Printf(" Rate limit: %.1f req/sec", cfg.RPS)
|
||||
log.Printf(" Delay: %d-%d ms", cfg.MinDelayMs, cfg.MaxDelayMs)
|
||||
log.Printf(" Retry: %d attempts", cfg.Retry)
|
||||
if cfg.Proxy != "" {
|
||||
log.Printf(" Proxy: %s", cfg.Proxy)
|
||||
}
|
||||
if cfg.Limit > 0 {
|
||||
log.Printf(" Limit: %d stocks", cfg.Limit)
|
||||
}
|
||||
if cfg.Resume {
|
||||
log.Printf(" Resume: enabled")
|
||||
}
|
||||
|
||||
db, err := msn.NewStockDB(cfg.DB)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open database: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
stocks := getStocksByIndex(cfg.Index)
|
||||
if cfg.Limit > 0 && cfg.Limit < len(stocks) {
|
||||
limited := make(map[string]msn.IDXStock)
|
||||
count := 0
|
||||
for ticker, stock := range stocks {
|
||||
if count >= cfg.Limit {
|
||||
break
|
||||
}
|
||||
limited[ticker] = stock
|
||||
count++
|
||||
}
|
||||
stocks = limited
|
||||
}
|
||||
|
||||
log.Printf("Stock list: %d stocks from '%s' index", len(stocks), cfg.Index)
|
||||
|
||||
var runID int64
|
||||
if cfg.Resume {
|
||||
runID, err = db.GetLastRunID()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get last run: %w", err)
|
||||
}
|
||||
if runID > 0 {
|
||||
log.Printf("Resuming run #%d", runID)
|
||||
} else {
|
||||
log.Printf("No incomplete run found, starting fresh")
|
||||
cfg.Resume = false
|
||||
}
|
||||
}
|
||||
|
||||
if !cfg.Resume {
|
||||
cfgMap := map[string]any{
|
||||
"index": cfg.Index,
|
||||
"concurrency": cfg.Concurrency,
|
||||
"rps": cfg.RPS,
|
||||
"delay": fmt.Sprintf("%d-%d", cfg.MinDelayMs, cfg.MaxDelayMs),
|
||||
"retry": cfg.Retry,
|
||||
"proxy": cfg.Proxy != "",
|
||||
}
|
||||
runID, err = db.StartScrapeRun(len(stocks), cfgMap)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start run: %w", err)
|
||||
}
|
||||
log.Printf("Started run #%d", runID)
|
||||
if err := db.InitProgress(runID, stocks); err != nil {
|
||||
return fmt.Errorf("failed to init progress: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
pendingStocks, err := db.GetPendingStocks(runID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get pending stocks: %w", err)
|
||||
}
|
||||
log.Printf("Pending: %d stocks to process", len(pendingStocks))
|
||||
if len(pendingStocks) == 0 {
|
||||
log.Println("No pending stocks, run complete")
|
||||
return nil
|
||||
}
|
||||
|
||||
rateLimiter := msn.NewRateLimiter(msn.RateLimiterConfig{RequestsPerSecond: cfg.RPS, MinDelayMs: cfg.MinDelayMs, MaxDelayMs: cfg.MaxDelayMs})
|
||||
client := msn.NewMSNClientWithConfig(msn.MSNClientConfig{Proxy: cfg.Proxy, RateLimiter: rateLimiter})
|
||||
defer client.Close()
|
||||
|
||||
type workItem struct{ ID, Ticker string }
|
||||
workChan := make(chan workItem, len(pendingStocks))
|
||||
for _, s := range pendingStocks {
|
||||
workChan <- workItem{ID: s.ID, Ticker: s.Ticker}
|
||||
}
|
||||
close(workChan)
|
||||
|
||||
var processed, successful, failed int
|
||||
total := len(pendingStocks)
|
||||
startTime := time.Now()
|
||||
|
||||
statusTicker := time.NewTicker(5 * time.Second)
|
||||
defer statusTicker.Stop()
|
||||
go func() {
|
||||
for range statusTicker.C {
|
||||
elapsed := time.Since(startTime)
|
||||
rate := float64(processed) / elapsed.Seconds()
|
||||
remaining := total - processed
|
||||
eta := time.Duration(float64(remaining)/rate) * time.Second
|
||||
log.Printf("Progress: %d/%d (%.1f%%) | Success: %d | Failed: %d | Rate: %.1f/s | ETA: %s",
|
||||
processed, total, float64(processed)*100/float64(total), successful, failed, rate, eta.Round(time.Second))
|
||||
}
|
||||
}()
|
||||
|
||||
done := make(chan bool)
|
||||
results := make(chan struct {
|
||||
ticker string
|
||||
success bool
|
||||
apis int
|
||||
err string
|
||||
}, cfg.Concurrency)
|
||||
|
||||
for w := 0; w < cfg.Concurrency; w++ {
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case work, ok := <-workChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
db.MarkProgressStarted(runID, work.ID)
|
||||
var stockData *msn.StockData
|
||||
var fetchErr error
|
||||
for attempt := 0; attempt <= cfg.Retry; attempt++ {
|
||||
stockData, fetchErr = client.FetchStockData(work.ID)
|
||||
if fetchErr == nil {
|
||||
break
|
||||
}
|
||||
if attempt < cfg.Retry {
|
||||
time.Sleep(time.Duration(500*(attempt+1)) * time.Millisecond)
|
||||
}
|
||||
}
|
||||
apisSuccess, apisFailed := 0, 0
|
||||
if stockData != nil {
|
||||
for _, status := range stockData.FetchStatus {
|
||||
if status == "ok" {
|
||||
apisSuccess++
|
||||
} else {
|
||||
apisFailed++
|
||||
}
|
||||
}
|
||||
if err := db.SaveStockData(stockData); err != nil {
|
||||
fetchErr = fmt.Errorf("save failed: %w", err)
|
||||
}
|
||||
}
|
||||
status := "success"
|
||||
errMsg := ""
|
||||
if fetchErr != nil || apisSuccess == 0 {
|
||||
status = "failed"
|
||||
if fetchErr != nil {
|
||||
errMsg = fetchErr.Error()
|
||||
}
|
||||
}
|
||||
db.UpdateProgress(runID, work.ID, status, apisSuccess, apisFailed, errMsg)
|
||||
results <- struct {
|
||||
ticker string
|
||||
success bool
|
||||
apis int
|
||||
err string
|
||||
}{work.Ticker, status == "success", apisSuccess, errMsg}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
go func() {
|
||||
for processed < total {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
done <- false
|
||||
return
|
||||
case r := <-results:
|
||||
processed++
|
||||
if r.success {
|
||||
successful++
|
||||
log.Printf("[%d/%d] %s - %d APIs succeeded", processed, total, r.ticker, r.apis)
|
||||
} else {
|
||||
failed++
|
||||
log.Printf("[%d/%d] %s - FAILED: %s", processed, total, r.ticker, r.err)
|
||||
}
|
||||
}
|
||||
}
|
||||
done <- true
|
||||
}()
|
||||
|
||||
completed := <-done
|
||||
elapsed := time.Since(startTime)
|
||||
if completed {
|
||||
db.CompleteScrapeRun(runID, "completed")
|
||||
log.Printf("\n=== Run #%d Completed ===", runID)
|
||||
} else {
|
||||
db.CompleteScrapeRun(runID, "interrupted")
|
||||
log.Printf("\n=== Run #%d Interrupted ===", runID)
|
||||
}
|
||||
log.Printf("Total: %d | Success: %d | Failed: %d", processed, successful, failed)
|
||||
log.Printf("Duration: %s | Rate: %.1f stocks/sec", elapsed.Round(time.Second), float64(processed)/elapsed.Seconds())
|
||||
log.Printf("Database: %s", cfg.DB)
|
||||
|
||||
staleStocks, _ := db.GetStaleStocks(7)
|
||||
if len(staleStocks) > 0 {
|
||||
log.Printf("\nWarning: %d stocks not seen in 7+ days:", len(staleStocks))
|
||||
for i, ticker := range staleStocks {
|
||||
if i >= 10 {
|
||||
log.Printf(" ... and %d more", len(staleStocks)-10)
|
||||
break
|
||||
}
|
||||
log.Printf(" - %s", ticker)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Helpers
|
||||
|
||||
func requireValue(args []string, i int, flag string) (string, int, error) {
|
||||
if i+1 >= len(args) {
|
||||
return "", i, fmt.Errorf("%s requires a value", flag)
|
||||
}
|
||||
return args[i+1], i + 1, nil
|
||||
}
|
||||
|
||||
func appendCSV(dst []string, csv string, upper bool) []string {
|
||||
for _, part := range strings.Split(csv, ",") {
|
||||
v := strings.TrimSpace(part)
|
||||
if upper {
|
||||
v = strings.ToUpper(v)
|
||||
}
|
||||
if v != "" {
|
||||
dst = append(dst, v)
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func dedupe(values []string) []string {
|
||||
seen := make(map[string]bool, len(values))
|
||||
out := make([]string, 0, len(values))
|
||||
for _, v := range values {
|
||||
if !seen[v] {
|
||||
seen[v] = true
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseDelay(v string) (int, int, error) {
|
||||
parts := strings.Split(v, "-")
|
||||
if len(parts) != 2 {
|
||||
return 0, 0, fmt.Errorf("invalid --delay value (use format min-max)")
|
||||
}
|
||||
minDelay, err := strconv.Atoi(parts[0])
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("invalid --delay min value: %w", err)
|
||||
}
|
||||
maxDelay, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("invalid --delay max value: %w", err)
|
||||
}
|
||||
if minDelay > maxDelay {
|
||||
return 0, 0, fmt.Errorf("invalid --delay value: min must be <= max")
|
||||
}
|
||||
return minDelay, maxDelay, nil
|
||||
}
|
||||
|
||||
func wantsHelp(args []string) bool {
|
||||
for _, a := range args {
|
||||
if a == "-h" || a == "--help" || a == "help" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func printMSNScreenerUsage() {
|
||||
fmt.Fprintf(os.Stderr, `Usage: rubick msn screener [options]
|
||||
|
||||
Options:
|
||||
--region <code> Country code (default: id)
|
||||
--filter <preset> top-performers|worst-performers|high-dividend|low-pe|52w-high|52w-low|high-volume|large-cap
|
||||
--limit <n> Max results (default: 50)
|
||||
--output, -o <file> Output JSON path
|
||||
`)
|
||||
}
|
||||
|
||||
func printMSNFetchUsage() {
|
||||
fmt.Fprintf(os.Stderr, `Usage: rubick msn fetch [options]
|
||||
|
||||
Options:
|
||||
--input <file> Screener JSON input
|
||||
--ids <id1,id2,...> Comma-separated MSN IDs
|
||||
--tickers <T1,T2,...> Comma-separated ticker symbols
|
||||
--concurrency <n> Parallel workers (default: 5)
|
||||
--output, -o <file> Output JSON path
|
||||
`)
|
||||
}
|
||||
|
||||
func printMSNFetchAllUsage() {
|
||||
fmt.Fprintf(os.Stderr, `Usage: rubick msn fetch-all [options]
|
||||
|
||||
Options:
|
||||
--db <file> SQLite database path (default: output/stocks.db)
|
||||
--index <name> all|lq45|idx30|idx80 (default: all)
|
||||
--proxy <url> Proxy URL (http://, https://, socks5://)
|
||||
--concurrency <n> Parallel workers (default: 5)
|
||||
--rps <n> Max requests/sec (default: 25)
|
||||
--delay <min-max> Random delay ms (default: 100-500)
|
||||
--retry <n> Retry attempts (default: 2)
|
||||
--limit <n> Process only N stocks
|
||||
--resume Resume incomplete run
|
||||
`)
|
||||
}
|
||||
|
||||
func printMSNLookupUsage() {
|
||||
fmt.Fprintf(os.Stderr, `Usage: rubick msn lookup <ticker1> [ticker2] ...
|
||||
`)
|
||||
}
|
||||
|
||||
func getStocksByIndex(index string) map[string]msn.IDXStock {
|
||||
allStocks := msn.GetAllIDXStocks()
|
||||
switch index {
|
||||
case "all":
|
||||
return allStocks
|
||||
case "lq45":
|
||||
return filterStocks(allStocks, []string{
|
||||
"ACES", "ADRO", "AKRA", "AMMN", "AMRT", "ANTM", "ASII", "BBCA",
|
||||
"BBNI", "BBRI", "BBTN", "BMRI", "BRPT", "BUKA", "CPIN", "EMTK",
|
||||
"ESSA", "EXCL", "GGRM", "GOTO", "HRUM", "ICBP", "INCO", "INDF",
|
||||
"INKP", "INTP", "ITMG", "KLBF", "MAPI", "MBMA", "MDKA", "MEDC",
|
||||
"PGAS", "PGEO", "PTBA", "SIDO", "SMGR", "TBIG", "TINS", "TLKM",
|
||||
"TOWR", "UNTR", "UNVR", "WIKA",
|
||||
})
|
||||
case "idx30":
|
||||
return filterStocks(allStocks, []string{
|
||||
"ADRO", "AMRT", "ANTM", "ASII", "BBCA", "BBNI", "BBRI", "BMRI",
|
||||
"BRPT", "CPIN", "EMTK", "EXCL", "GOTO", "ICBP", "INCO", "INDF",
|
||||
"ITMG", "KLBF", "MDKA", "MEDC", "PGAS", "PTBA", "SMGR", "TBIG",
|
||||
"TINS", "TLKM", "TOWR", "UNTR", "UNVR",
|
||||
})
|
||||
case "idx80":
|
||||
return filterStocks(allStocks, []string{
|
||||
"ACES", "ADRO", "AGII", "AKRA", "AMMN", "AMRT", "ANTM", "ARTO",
|
||||
"ASII", "BBCA", "BBNI", "BBRI", "BBTN", "BFIN", "BMRI", "BRPT",
|
||||
"BSDE", "BTPS", "BUKA", "CPIN", "CTRA", "DMAS", "EMTK", "ERAA",
|
||||
"ESSA", "EXCL", "GGRM", "GOTO", "HEAL", "HMSP", "HRUM", "ICBP",
|
||||
"INCO", "INDF", "INKP", "INTP", "ITMG", "JPFA", "JSMR", "KLBF",
|
||||
"LPKR", "LPPF", "MAPI", "MBMA", "MDKA", "MEDC", "MIKA", "MNCN",
|
||||
"PGAS", "PGEO", "PNBN", "PTBA", "PTPP", "PWON", "SCMA", "SIDO",
|
||||
"SMGR", "SMRA", "SRTG", "TAPG", "TBIG", "TINS", "TKIM", "TLKM",
|
||||
"TOWR", "TPIA", "UNTR", "UNVR", "WIKA", "WSKT",
|
||||
})
|
||||
default:
|
||||
log.Printf("Unknown index '%s', using all stocks", index)
|
||||
return allStocks
|
||||
}
|
||||
}
|
||||
|
||||
func filterStocks(all map[string]msn.IDXStock, tickers []string) map[string]msn.IDXStock {
|
||||
result := make(map[string]msn.IDXStock)
|
||||
for _, ticker := range tickers {
|
||||
if stock, ok := all[ticker]; ok {
|
||||
result[ticker] = stock
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
336
internal/cli/news_cli.go
Normal file
336
internal/cli/news_cli.go
Normal file
|
|
@ -0,0 +1,336 @@
|
|||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/enetx/g"
|
||||
"github.com/enetx/surf"
|
||||
"github.com/joho/godotenv"
|
||||
)
|
||||
|
||||
type NewsConfig struct {
|
||||
Query string
|
||||
From time.Time
|
||||
To time.Time
|
||||
Count int
|
||||
Concurrency int
|
||||
Output string
|
||||
StockMode bool
|
||||
}
|
||||
|
||||
type EnrichedResult struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Description string `json:"description"`
|
||||
PageAge string `json:"page_age"`
|
||||
Text string `json:"text"`
|
||||
FetchStatus string `json:"fetch_status"`
|
||||
ExtractStatus string `json:"extract_status"`
|
||||
}
|
||||
|
||||
type OutputData struct {
|
||||
Query string `json:"query"`
|
||||
GeneratedAt string `json:"generated_at"`
|
||||
Results []EnrichedResult `json:"results"`
|
||||
}
|
||||
|
||||
func runNewsCommand(args []string) int {
|
||||
godotenv.Load()
|
||||
|
||||
if len(args) == 0 {
|
||||
printNewsUsage()
|
||||
return 1
|
||||
}
|
||||
if args[0] == "-h" || args[0] == "--help" || args[0] == "help" {
|
||||
printNewsUsage()
|
||||
return 0
|
||||
}
|
||||
|
||||
cfg, err := parseNewsArgs(args)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
if err := executeNews(cfg); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func executeNews(config NewsConfig) error {
|
||||
finalQuery := config.Query
|
||||
if config.StockMode {
|
||||
terms := strings.Split(config.Query, ",")
|
||||
for i := range terms {
|
||||
terms[i] = strings.TrimSpace(terms[i])
|
||||
}
|
||||
finalQuery = BuildStockQuery(terms...)
|
||||
log.Printf("Stock mode query: %s", finalQuery)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
defer signal.Stop(sigChan)
|
||||
go func() {
|
||||
<-sigChan
|
||||
log.Println("Shutting down...")
|
||||
cancel()
|
||||
}()
|
||||
|
||||
client := surf.NewClient().Builder().Impersonate().Chrome().Build().Unwrap()
|
||||
defer client.CloseIdleConnections()
|
||||
|
||||
log.Printf("Searching for: %s", finalQuery)
|
||||
log.Printf("Date range: %s to %s, count: %d", config.From.Format("2006-01-02"), config.To.Format("2006-01-02"), config.Count)
|
||||
|
||||
searchResults, err := SearchBrave(client, SearchConfig{Query: finalQuery, From: config.From, To: config.To, Count: config.Count})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to search: %w", err)
|
||||
}
|
||||
log.Printf("Found %d results", len(searchResults))
|
||||
if len(searchResults) == 0 {
|
||||
log.Println("No results found, exiting")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Println("Starting Python extractor...")
|
||||
extractor, err := NewExtractor(config.Concurrency)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start extractor: %w", err)
|
||||
}
|
||||
defer extractor.Close()
|
||||
log.Println("Python extractor ready")
|
||||
|
||||
results := processURLs(ctx, client, extractor, searchResults, config.Concurrency)
|
||||
output := OutputData{Query: finalQuery, GeneratedAt: time.Now().UTC().Format(time.RFC3339), Results: results}
|
||||
if err := saveOutput(output, config.Output); err != nil {
|
||||
return fmt.Errorf("failed to save output: %w", err)
|
||||
}
|
||||
|
||||
successCount := 0
|
||||
for _, r := range results {
|
||||
if r.ExtractStatus == "ok" {
|
||||
successCount++
|
||||
}
|
||||
}
|
||||
log.Printf("Output saved to %s", config.Output)
|
||||
log.Printf("Successfully extracted %d/%d articles", successCount, len(results))
|
||||
return nil
|
||||
}
|
||||
|
||||
func printNewsUsage() {
|
||||
fmt.Fprintf(os.Stderr, `Usage: rubick news <query> [options]
|
||||
|
||||
Arguments:
|
||||
<query> Search query (required)
|
||||
For --stock mode: comma-separated stock terms
|
||||
|
||||
Options:
|
||||
--from <date> Start date in YYYY-MM-DD format (default: 7 days ago)
|
||||
--to <date> End date in YYYY-MM-DD format (default: today)
|
||||
--count <n> Number of results to fetch (default: 20)
|
||||
--concurrency <n> Number of parallel workers (default: 10)
|
||||
--output, -o <file> Output file path (default: output_YYYYMMDD.json)
|
||||
--stock Auto-builds IDX-focused boolean query
|
||||
|
||||
Environment:
|
||||
BRAVE_API_KEY Brave Search API key
|
||||
|
||||
Examples:
|
||||
rubick news "IHSG stock market"
|
||||
rubick news "BBCA,Bank Central Asia" --stock --from 2026-02-01 --to 2026-02-10
|
||||
`)
|
||||
}
|
||||
|
||||
func parseNewsArgs(args []string) (NewsConfig, error) {
|
||||
query := args[0]
|
||||
args = args[1:]
|
||||
|
||||
now := time.Now()
|
||||
cfg := NewsConfig{
|
||||
Query: query,
|
||||
From: now.AddDate(0, 0, -7),
|
||||
To: now,
|
||||
Count: 20,
|
||||
Concurrency: 10,
|
||||
Output: fmt.Sprintf("output_%s.json", now.Format("20060102")),
|
||||
}
|
||||
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch args[i] {
|
||||
case "--from":
|
||||
v, n, err := requireValue(args, i, "--from")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
t, err := time.Parse("2006-01-02", v)
|
||||
if err != nil {
|
||||
return cfg, fmt.Errorf("invalid --from date: %w", err)
|
||||
}
|
||||
cfg.From = t
|
||||
i = n
|
||||
case "--to":
|
||||
v, n, err := requireValue(args, i, "--to")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
t, err := time.Parse("2006-01-02", v)
|
||||
if err != nil {
|
||||
return cfg, fmt.Errorf("invalid --to date: %w", err)
|
||||
}
|
||||
cfg.To = t
|
||||
i = n
|
||||
case "--count":
|
||||
v, n, err := requireValue(args, i, "--count")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
nval, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return cfg, fmt.Errorf("invalid --count value: %w", err)
|
||||
}
|
||||
cfg.Count = nval
|
||||
i = n
|
||||
case "--concurrency":
|
||||
v, n, err := requireValue(args, i, "--concurrency")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
nval, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return cfg, fmt.Errorf("invalid --concurrency value: %w", err)
|
||||
}
|
||||
cfg.Concurrency = nval
|
||||
i = n
|
||||
case "--output", "-o":
|
||||
v, n, err := requireValue(args, i, "--output")
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
cfg.Output = v
|
||||
i = n
|
||||
case "--stock":
|
||||
cfg.StockMode = true
|
||||
default:
|
||||
return cfg, fmt.Errorf("unknown option: %s", args[i])
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.Count < 1 {
|
||||
return cfg, fmt.Errorf("--count must be >= 1")
|
||||
}
|
||||
if cfg.Concurrency < 1 {
|
||||
return cfg, fmt.Errorf("--concurrency must be >= 1")
|
||||
}
|
||||
if cfg.From.After(cfg.To) {
|
||||
return cfg, fmt.Errorf("--from must be on or before --to")
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func processURLs(ctx context.Context, client *surf.Client, extractor *Extractor, searchResults []BraveResult, concurrency int) []EnrichedResult {
|
||||
results := make([]EnrichedResult, len(searchResults))
|
||||
for i, sr := range searchResults {
|
||||
results[i] = EnrichedResult{Title: sr.Title, URL: sr.URL, Description: sr.Description, PageAge: sr.PageAge, FetchStatus: "pending", ExtractStatus: "pending"}
|
||||
}
|
||||
|
||||
work := make(chan int, len(searchResults))
|
||||
for i := range searchResults {
|
||||
work <- i
|
||||
}
|
||||
close(work)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
for w := 0; w < concurrency; w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case idx, ok := <-work:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
result := processURL(ctx, client, extractor, searchResults[idx])
|
||||
mu.Lock()
|
||||
results[idx] = result
|
||||
mu.Unlock()
|
||||
log.Printf("[%d/%d] %s - fetch: %s, extract: %s", idx+1, len(searchResults), truncate(result.URL, 50), result.FetchStatus, result.ExtractStatus)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return results
|
||||
}
|
||||
|
||||
func processURL(ctx context.Context, client *surf.Client, extractor *Extractor, sr BraveResult) EnrichedResult {
|
||||
result := EnrichedResult{Title: sr.Title, URL: sr.URL, Description: sr.Description, PageAge: sr.PageAge}
|
||||
if err := ctx.Err(); err != nil {
|
||||
result.FetchStatus = "cancelled"
|
||||
result.ExtractStatus = "skipped"
|
||||
return result
|
||||
}
|
||||
resp := client.Get(g.String(sr.URL)).Do()
|
||||
if resp.IsErr() {
|
||||
result.FetchStatus = "failed"
|
||||
result.ExtractStatus = "skipped"
|
||||
return result
|
||||
}
|
||||
r := resp.Ok()
|
||||
if r.StatusCode != 200 {
|
||||
result.FetchStatus = "failed"
|
||||
result.ExtractStatus = "skipped"
|
||||
return result
|
||||
}
|
||||
html := r.Body.String().Ok().Std()
|
||||
result.FetchStatus = "ok"
|
||||
extractResp, err := extractor.Extract(ctx, sr.URL, html)
|
||||
if err != nil {
|
||||
result.ExtractStatus = "failed"
|
||||
return result
|
||||
}
|
||||
result.Text = extractResp.Text
|
||||
result.ExtractStatus = extractResp.Status
|
||||
return result
|
||||
}
|
||||
|
||||
func saveOutput(output OutputData, filename string) error {
|
||||
data, err := json.MarshalIndent(output, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir := filepath.Dir(filename); dir != "." {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return os.WriteFile(filename, data, 0o644)
|
||||
}
|
||||
|
||||
func truncate(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen-3] + "..."
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue