16 KiB
16 KiB
You are a technical documentation assistant specializing in creating clear, concise PR messages. Given a git diff, you will:
-
Analyze the structural changes to identify:
- New components/interfaces
- Architectural changes
- Behavioral modifications
- Impact on existing systems
-
Create a PR message following this format: ():
[PROSE DESCRIPTION explaining the changes and their purpose]
[OPTIONAL SECTIONS like "Structs:", "Interfaces:", "Methods:", etc. when relevant]
Each section should:
- Use bullet points for clarity
- Bold key terms
- Include brief descriptions
- Focus on significant changes
- Exclude trivial details (imports, formatting)
INPUT:
diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go
index 9aee19c..9863f0b 100644
--- a/pkg/cli/cli.go
+++ b/pkg/cli/cli.go
@@ -1,21 +1,13 @@
package cli
import (
- "context"
- "net/http"
"strings"
- "time"
"github.com/sanity-io/litter"
"github.com/spf13/cobra"
- "github.com/org/Core/pkg/logging"
"github.com/org/Core/pkg/structs"
- "github.com/org/example/internal/storage"
- "github.com/org/example/pkg/browser"
"github.com/org/example/pkg/config"
- "github.com/org/example/pkg/crawler"
"github.com/org/example/pkg/errors"
- "github.com/org/example/pkg/models"
cookieflags "github.com/org/example/pkg/plugins/cookieflags"
cookiescope "github.com/org/example/pkg/plugins/cookiescope"
corsmisconfig "github.com/org/example/pkg/plugins/cors"
@@ -124,7 +116,18 @@ func NewRootCmd() (*cobra.Command, error) {
logger.Tracef("Crawler Configuration:
=============
%s
=============
", litter.Sdump(crawlConfig))
logger.Tracef("Vulnerability Scan Configuration:
=============
%s
=============
", litter.Sdump(pluginsConfig))
- return StartScan(sysConfig, crawlConfig, pluginsConfig)
+ manager, err := NewManagerBuilder(sysConfig, crawlConfig, pluginsConfig).Build()
+
+ if err != nil {
+ return err
+ }
+
+ err = manager.Start()
+ if err != nil {
+ return err
+ }
+
+ return manager.Stop()
},
}
@@ -186,113 +189,6 @@ func parseConfigFiles(crawlConfPath, systemConfPath, pluginConfPath string) (*co
return crawlConfig, sysConfig, vulnConfig, nil
}
-func StartScan(sysCfg *config.SystemConfiguration, //nolint:cyclop
- crawlCfg *config.CrawlConfiguration,
- pluginCfg *config.PluginsConfiguration,
-) error {
- // Initialize shared components
- logger := logging.NewLoggerBuilder().Build()
- metrics := &models.Metrics{}
- // Initialize the browser session
- browserSession, err := browser.New(logger, crawlCfg)
- if err != nil {
- return errors.ErrFailedExecution.WrapWithNoMessage(err)
- }
- defer browserSession.Close()
- if err := browserSession.Start(context.Background()); err != nil {
- return errors.ErrFailedExecution.WrapWithNoMessage(err)
- }
- // Create custom HTTP client if needed
- // TODO: Change and centralise
- // see https://github.com/org/example/issues/436
- customClient := &http.Client{
- Timeout: time.Second * 30,
- // Add other custom configurations...
- }
- // Iniialize storage once
- scanStorage, err := storage.NewScanStorage(sysCfg, metrics, logger)
- if err != nil {
- return errors.ErrStorageSetup.WrapWithNoMessage(err)
- }
- // Start batch session before any operations
- err = scanStorage.BatchSession.Start()
- if err != nil {
- return errors.ErrStorageSetup.WrapWithNoMessage(err)
- }
- // Initialize crawler with all options
- crawler, err := crawler.New(
- crawlCfg,
- sysCfg,
- crawler.WithStorage(scanStorage.Database, scanStorage.BatchSession),
- crawler.WithLogger(logger),
- crawler.WithMetrics(metrics),
- crawler.WithHTTPClient(customClient),
- crawler.WithBrowser(browserSession),
- )
- if err != nil {
- return errors.ErrFailedExecution.WrapWithNoMessage(err)
- }
- // Initialize scanner with shared components
- scan, err := scanner.NewScanner(
- sysCfg,
- crawlCfg,
- pluginCfg,
- browserSession,
- scanner.WithStorage(scanStorage),
- scanner.WithLogger(logger),
- scanner.WithHTTPClient(customClient),
- )
- if err != nil {
- return errors.ErrFailedExecution.WrapWithNoMessage(err)
- }
- err = initializePluginsFromConfig(scan, pluginCfg)
- if err != nil {
- return errors.ErrInitialisePlugin.WrapWithNoMessage(err)
- }
- output, err := crawler.Start()
- if err != nil {
- crawler.Close()
- return errors.ErrFailedExecution.WrapWithNoMessage(err)
- }
- // Add this: Stop scanner before waiting for batch operations
- scan.Stop()
- logger.Debugf("Crawl completed with metrics: %+v", output.Metrics)
- if sysCfg.ShouldExportMetrics() {
- err := output.Metrics.ToJSONFile()
- if err != nil {
- return errors.ErrJSONMarshalling.WrapWithNoMessage(err)
- }
- }
- if output.BenchmarkResults != nil {
- logger.Debugf("Benchmark results: %+v", output.BenchmarkResults)
- }
- scanStorage.BatchSession.Wait()
- err = scanStorage.BatchSession.Stop()
- if err != nil {
- return errors.ErrStorageSetup.WrapWithNoMessage(err)
- }
- crawler.Close()
- return nil
-}
// BuildPluginsFromConfig creates plugin instances based on configuration
func BuildPluginsFromConfig(config *config.PluginsConfiguration) []*structs.Plugin {
enabledPlugins := []*structs.Plugin{}
diff --git a/pkg/cli/interface.go b/pkg/cli/interface.go
new file mode 100644
index 0000000..4d68a45
--- /dev/null
+++ b/pkg/cli/interface.go
@@ -0,0 +1,10 @@
+package cli
+
+// Scanner represents the core scanning operations lifecycle
+type ScanManagerInterface interface {
+ // Start initializes and begins the scanning process
+ Start() error
+
+ // Stop gracefully terminates all scanning operations
+ Stop() error
+}
diff --git a/pkg/cli/manager.go b/pkg/cli/manager.go
new file mode 100644
index 0000000..3f2a8fc
--- /dev/null
+++ b/pkg/cli/manager.go
@@ -0,0 +1,277 @@
+package cli
+
+import (
+ "context"
+ "net/http"
+
+ "github.com/org/Core/pkg/logging"
+ "github.com/org/example/internal/storage"
+ "github.com/org/example/pkg/browser"
+ "github.com/org/example/pkg/config"
+ "github.com/org/example/pkg/crawler"
+ "github.com/org/example/pkg/errors"
+ "github.com/org/example/pkg/models"
+ "github.com/org/example/pkg/scanner"
+)
+
+var _ ScanManagerInterface = (*ScanManager)(nil)
+
+type ScanManager struct {
+ browser *browser.Session
+ crawler *crawler.Crawler
+ scanner *scanner.Scanner
+ storage *storage.ScanStorage
+ logger *logging.Logger
+ metrics *models.Metrics
+ httpClient *http.Client
+
+ sysCfg *config.SystemConfiguration
+ crawlCfg *config.CrawlConfiguration
+ pluginCfg *config.PluginsConfiguration
+}
+
+// ScanManagerBuilder handles the construction of a ScanManager
+type ScanManagerBuilder struct {
+ manager *ScanManager
+ err error
+}
+
+// NewManagerBuilder creates a new builder for ScanManager
+func NewManagerBuilder(sysCfg *config.SystemConfiguration,
+ crawlCfg *config.CrawlConfiguration,
+ pluginCfg *config.PluginsConfiguration,
+) *ScanManagerBuilder {
+ builder := &ScanManagerBuilder{
+ manager: &ScanManager{
+ logger: logging.NewLoggerBuilder().Build(),
+ metrics: &models.Metrics{},
+ httpClient: &http.Client{},
+ },
+ }
+
+ if sysCfg == nil || crawlCfg == nil || pluginCfg == nil {
+ builder.err = errors.ErrInvalidArgument.New("configurations cannot be nil")
+
+ return builder
+ }
+
+ builder.manager.sysCfg = sysCfg
+ builder.manager.crawlCfg = crawlCfg
+ builder.manager.pluginCfg = pluginCfg
+
+ return builder
+}
+
+// WithLogger sets a custom logger
+func (b *ScanManagerBuilder) WithLogger(logger *logging.Logger) *ScanManagerBuilder {
+ if b.err != nil {
+ return b
+ }
+
+ if logger == nil {
+ b.err = errors.ErrInvalidArgument.New("logger cannot be nil")
+
+ return b
+ }
+
+ b.manager.logger = logger
+
+ return b
+}
+
+// WithMetrics sets custom metrics
+func (b *ScanManagerBuilder) WithMetrics(metrics *models.Metrics) *ScanManagerBuilder {
+ if b.err != nil {
+ return b
+ }
+
+ if metrics == nil {
+ b.err = errors.ErrInvalidArgument.New("metrics cannot be nil")
+
+ return b
+ }
+
+ b.manager.metrics = metrics
+
+ return b
+}
+
+// Build creates the ScanManager instance
+func (b *ScanManagerBuilder) Build() (*ScanManager, error) {
+ if b.err != nil {
+ return nil, b.err
+ }
+
+ return b.manager, nil
+}
+
+func (sm *ScanManager) initBrowser(crawlCfg *config.CrawlConfiguration) error {
+ var err error
+
+ sm.browser, err = browser.New(sm.logger, crawlCfg)
+ if err != nil {
+ return errors.ErrFailedExecution.WrapWithNoMessage(err)
+ }
+
+ if err := sm.browser.Start(context.Background()); err != nil {
+ sm.browser.Close()
+
+ return errors.ErrFailedExecution.WrapWithNoMessage(err)
+ }
+
+ return nil
+}
+
+func (sm *ScanManager) initStorage(sysCfg *config.SystemConfiguration) error {
+ var err error
+
+ storage, err := storage.NewScanStorage(sysCfg, sm.metrics, sm.logger)
+ if err != nil {
+ return errors.ErrStorageSetup.WrapWithNoMessage(err)
+ }
+
+ sm.storage = &storage
+
+ err = sm.storage.BatchSession.Start()
+ if err != nil {
+ return errors.ErrStorageSetup.WrapWithNoMessage(err)
+ }
+
+ return nil
+}
+
+func (sm *ScanManager) initCrawler(sysCfg *config.SystemConfiguration, crawlCfg *config.CrawlConfiguration) error {
+ var err error
+
+ sm.crawler, err = crawler.New(
+ crawlCfg,
+ sysCfg,
+ crawler.WithStorage(sm.storage.Database, sm.storage.BatchSession),
+ crawler.WithLogger(sm.logger),
+ crawler.WithMetrics(sm.metrics),
+ crawler.WithHTTPClient(sm.httpClient),
+ crawler.WithBrowser(sm.browser),
+ )
+
+ if err != nil {
+ return errors.ErrFailedExecution.WrapWithNoMessage(err)
+ }
+
+ return nil
+}
+
+func (sm *ScanManager) initScanner() error {
+ var err error
+
+ sm.scanner, err = scanner.NewScanner(
+ sm.sysCfg,
+ sm.crawlCfg,
+ sm.pluginCfg,
+ sm.browser,
+ scanner.WithStorage(*sm.storage),
+ scanner.WithLogger(sm.logger),
+ scanner.WithHTTPClient(sm.httpClient),
+ )
+
+ if err != nil {
+ return errors.ErrFailedExecution.WrapWithNoMessage(err)
+ }
+
+ err = initializePluginsFromConfig(sm.scanner, sm.pluginCfg)
+ if err != nil {
+ return errors.ErrInitialisePlugin.WrapWithNoMessage(err)
+ }
+
+ return nil
+}
+
+func (sm *ScanManager) cleanup() error {
+ var errs []error
+
+ if sm.crawler != nil {
+ sm.crawler.Close()
+ }
+
+ if sm.scanner != nil {
+ sm.scanner.Stop()
+ }
+
+ if sm.storage != nil && sm.storage.BatchSession != nil {
+ sm.storage.BatchSession.Wait()
+
+ err := sm.storage.BatchSession.Stop()
+ if err != nil {
+ errs = append(errs, errors.ErrStorageSetup.WrapWithNoMessage(err))
+ }
+ }
+
+ if sm.browser != nil {
+ sm.browser.Close()
+ }
+
+ if len(errs) > 0 {
+ return errors.ErrFailedExecution.New("multiple cleanup errors occurred: %v", errs)
+ }
+
+ return nil
+}
+
+func (sm *ScanManager) Start() error {
+ err := sm.initBrowser(sm.crawlCfg)
+ if err != nil {
+ return err
+ }
+
+ err = sm.initStorage(sm.sysCfg)
+ if err != nil {
+ _ = sm.cleanup()
+
+ return err
+ }
+
+ err = sm.initCrawler(sm.sysCfg, sm.crawlCfg)
+ if err != nil {
+ _ = sm.cleanup()
+
+ return err
+ }
+
+ err = sm.initScanner()
+ if err != nil {
+ _ = sm.cleanup()
+
+ return err
+ }
+
+ // Start the crawl
+ output, err := sm.crawler.Start()
+ if err != nil {
+ _ = sm.cleanup()
+
+ return errors.ErrFailedExecution.WrapWithNoMessage(err)
+ }
+
+ sm.logger.Debugf("Crawl completed with metrics: %+v", output.Metrics)
+
+ if sm.sysCfg.ShouldExportMetrics() {
+ err := output.Metrics.ToJSONFile()
+
+ if err != nil {
+ return errors.ErrJSONMarshalling.WrapWithNoMessage(err)
+ }
+ }
+
+ if output.BenchmarkResults != nil {
+ sm.logger.Debugf("Benchmark results: %+v", output.BenchmarkResults)
+ }
+
+ return nil
+}
+
+func (sm *ScanManager) Stop() error {
+ if sm == nil {
+ return nil
+ }
+
+ return sm.cleanup()
+}
diff --git a/pkg/scanner/scanner.go b/pkg/scanner/scanner.go
index c0a104f..6ef620a 100644
--- a/pkg/scanner/scanner.go
+++ b/pkg/scanner/scanner.go
@@ -676,6 +676,8 @@ func (s *Scanner) Stop() {
s.wg.Wait()
close(s.initialRequestQueue)
close(s.processedRequestQueue)
+
+ instance = nil
}
// Wait blocks until all workers have finished processing.
diff --git a/tests/e2e/wallace_test.go b/tests/e2e/wallace_test.go
index 0e899e9..b8de5c8 100644
--- a/tests/e2e/wallace_test.go
+++ b/tests/e2e/wallace_test.go
@@ -550,31 +550,26 @@ type scanResults struct {
// }
// TestPlugins verifies that each plugin detects at least one vulnerability for each test path
-func TestPlugins(t *testing.T) { //nolint: paralleltest
- // Retrieve all available plugins without specific configurations
+func TestPlugins(t *testing.T) {
plugins := cli.BuildAllPlugins()
if len(plugins) == 0 {
t.Fatal("No plugins available to test")
}
- // Ensure that there are test paths to scan
if len(SiteURLs) == 0 {
t.Fatal("No site URLs available for scanning")
}
- // Iterate over each plugin and perform the test
- for _, plugin := range plugins { //nolint: paralleltest
+ for _, plugin := range plugins {
pluginName := plugin.Name()
t.Run(pluginName, func(t *testing.T) {
- // t.Parallel()
// Setup test environment for the current plugin
resultDir, dbPath, cleanup := setupTestEnvironment(t, pluginName)
defer cleanup()
- // Initialize plugin-specific configuration if needed
- // Currently, using default configurations
+ // Initialize plugin-specific configuration
pluginConfig := coreStructs.NewPluginConfigBuilder(pluginName).
WithCustomConfiguration(coreStructs.CustomPluginConfig{"name": pluginName}).
Build()
@@ -595,7 +590,6 @@ func TestPlugins(t *testing.T) { //nolint: paralleltest
// Collect test URLs from initialized plugins' metadata
testURLs := collectTestURLs(pluginInstances)
- // Skip the test if no test URLs are defined for the plugin
if len(testURLs) == 0 {
t.Skipf("No test URLs defined for plugin '%s'. Skipping test.", pluginName)
}
@@ -606,11 +600,26 @@ func TestPlugins(t *testing.T) { //nolint: paralleltest
t.Fatalf("Failed to get default test config: %v", err)
}
- // Run the scan
- if err := cli.StartScan(sysConfig, crawlConfig, vulnConfig); err != nil {
+ // Create and start scanner using the new interface
+ scanner, err := cli.NewManagerBuilder(sysConfig, crawlConfig, vulnConfig).
+ Build()
+ if err != nil {
+ t.Fatalf("Failed to create scanner: %v", err)
+ }
+
+ // Start the scan
+ if err := scanner.Start(); err != nil {
t.Fatalf("Failed to start scan: %v", err)
}
+ // Ensure scanner is stopped after test
+ defer func() {
+ if err := scanner.Stop(); err != nil {
+ t.Errorf("Failed to stop scanner: %v", err)
+ }
+ }()
+
+ // Allow time for processing
time.Sleep(2 * time.Second)
// Verify results in the database
@@ -621,11 +630,11 @@ func TestPlugins(t *testing.T) { //nolint: paralleltest
// Assert that at least one vulnerability was found per test path
expectedVulns := len(testURLs)
- assert.GreaterOrEqual(t, vulnCount, expectedVulns, "Expected at least %d vulnerabilities", expectedVulns)
- time.Sleep(5 * time.Second)
+ assert.GreaterOrEqual(t, vulnCount, expectedVulns,
+ "Expected at least %d vulnerabilities", expectedVulns)
+
// Verify that result files are present
results := readScanResults(sysConfig.GetOutputDirectory())
assert.True(t, results.resultsFound, "Expected at least one result file")
})
}
OUTPUT:
refactor(cli): introduce ScanManager for scan lifecycle management
Introduced `ScanManager` which simplifies the scan initiation process and makes the scan modular (instead of a monolithic function that bubbles everything). The manager implements the `ScanManagerInterface`:
- **Start():** Initializes and starts the scan process.
- **Stop():** Gracefully terminates the scan process.
---
Structs:
- **ScanManager:** A new struct that manages the lifecycle of scanning components such as browser, crawler, scanner, storage, logger, and metrics.
- **ScanManagerBuilder:** A builder pattern implementation to construct a `ScanManager` with custom configurations.
Guidelines:
- Title should follow commit convention
- Description should be clear and business-focused
- Technical details should be organized in sections
- Use markdown formatting for readability
- Focus on architectural and behavioral changes
- Exclude trivial changes
- Keep descriptions concise but complete