diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..91b64be --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,29 @@ +when: + event: [push, pull_request] + +pipeline: + dependencies: + image: python:3.11-slim + commands: + - pip install -r requirements.txt + + syntax-check: + image: python:3.11-slim + commands: + - python -m py_compile app.py + - python -m py_compile import_from_aws.py + - python -m py_compile import_data.py + + docker-build: + image: docker:dind + volumes: + - /var/run/docker.sock:/var/run/docker.sock + commands: + - docker build -t sgo:${CI_COMMIT_SHA} . + + security-scan: + image: python:3.11-slim + commands: + - pip install bandit safety + - bandit -r . -ll || true + - safety check --file requirements.txt || true diff --git a/.woodpecker/docker-checks.yml b/.woodpecker/docker-checks.yml new file mode 100644 index 0000000..b236a4b --- /dev/null +++ b/.woodpecker/docker-checks.yml @@ -0,0 +1,19 @@ +pipeline: + docker-lint: + image: hadolint/hadolint:latest-alpine + commands: + - hadolint Dockerfile + + docker-build-test: + image: docker:dind + volumes: + - /var/run/docker.sock:/var/run/docker.sock + commands: + - docker build -t sgo:test . + - docker images sgo:test + + docker-compose-validate: + image: docker/compose:latest + commands: + - docker-compose config -q + - docker-compose -f docker-compose.local.yml config -q diff --git a/.woodpecker/python-checks.yml b/.woodpecker/python-checks.yml new file mode 100644 index 0000000..3ec329c --- /dev/null +++ b/.woodpecker/python-checks.yml @@ -0,0 +1,21 @@ +pipeline: + python-lint: + image: python:3.11-slim + commands: + - pip install flake8 pylint + - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + python-syntax: + image: python:3.11-slim + commands: + - python -m py_compile app.py + - python -m py_compile import_from_aws.py + - python -m py_compile import_data.py + + python-security: + image: python:3.11-slim + commands: + - pip install bandit + - bandit -r . -f json -o bandit-report.json || true + - bandit -r . -ll diff --git a/README.md b/README.md index 99ca471..75f43e9 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,17 @@ podman-compose up --build # 4. Select AWS profiles, enter MFA codes, and import! ``` +## ⚠️ Security Warning + +**This application is designed for LOCAL USE ONLY. Do NOT expose it to the internet.** + +- SGO has no authentication or authorization mechanisms +- It provides direct access to your AWS infrastructure data +- It reads AWS credentials from your local system +- Exposing it publicly would allow unauthorized access to sensitive AWS information + +**Always run on localhost (127.0.0.1) only. Never expose port 5000 to external networks.** + ## Features - **Direct AWS Import**: Import data directly from AWS using `~/.aws/config` with MFA/OTP support @@ -104,6 +115,190 @@ EOF - Better for development - Use `docker-compose.local.yml`: +```bash +docker-compose -f docker-compose.local.yml up --build +# or +podman-compose -f docker-compose.local.yml up --build +``` + +Or edit `docker-compose.yml` and swap the volume configuration as indicated in comments. + +### User/Group Configuration + +To avoid permission issues, set `PUID` and `PGID` to match your host user: + +```bash +# Find your IDs +id -u # Your PUID +id -g # Your PGID + +# Add to .env file +echo "PUID=$(id -u)" >> .env +echo "PGID=$(id -g)" >> .env +``` + +### Stopping the Application + +```bash +# Stop with Ctrl+C, or: +docker-compose down # Docker +podman-compose down # Podman + +# To also remove the data volume: +docker-compose down -v +``` + +## Quick Start (Local Python) + +If you prefer to run without containers: + +### 1. Install Dependencies + +```bash +python3 -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate +pip install -r requirements.txt +``` + +### 2. Start the Application + +```bash +python app.py +``` + +### 3. Open Browser + +Navigate to `http://localhost:5000` + +## Important Notes + +- **⚠️ LOCAL USE ONLY**: Never expose this application to the internet. It has no authentication and provides access to sensitive AWS data. +- **Database Persistence**: When using containers, the database persists in the `./data` directory +- **Session Caching**: AWS sessions are cached for 55 minutes, allowing multiple refreshes without re-authentication +- **Parallel Import**: All selected AWS accounts are imported simultaneously for maximum speed + + +## AWS Configuration + +### MFA Device Setup + +For profiles that require MFA, add your MFA device ARN to `~/.aws/config`: + +```ini +[profile nonprod-p1p2-admin] +region = us-west-2 +mfa_serial = arn:aws:iam::131340773912:mfa/your-username +``` + +### Finding Your MFA Device ARN + +1. Go to AWS IAM Console +2. Navigate to Users → Your User → Security Credentials +3. Copy the ARN from "Assigned MFA device" + +### How MFA Works in the GUI + +1. The import page shows all profiles from `~/.aws/config` +2. Select the profiles you want to import +3. Enter MFA codes in the text boxes (one per profile) +4. Click "Start Import" to begin +5. Real-time progress shows authentication and data fetching +6. MFA session is valid for 1 hour - refresh without re-entering codes during this window + +## Usage + +### Search + +1. Type in the search box (minimum 2 characters) +2. Results appear instantly as you type +3. Filter by resource type using the buttons: **All Resources** | **EC2 Instances** | **Security Groups** +4. **Enable Regex**: Check the "Regex" box to use regular expressions + - Example: `^prod-.*-\d+$` finds names starting with "prod-" and ending with numbers + - Example: `(dev|test|qa)` finds names containing dev, test, or qa + - Example: `10\.0\.\d+\.\d+` finds IP addresses in the 10.0.x.x range + +### View Details + +**EC2 Instance View:** +- Click on any EC2 instance from search results +- Main card shows EC2 details (Instance ID, IP, State, Account, Tags) +- Nested cards show all attached Security Groups with their details + +**Security Group View:** +- Click on any Security Group from search results +- Main card shows SG details (Group ID, Name, Ingress Rules, Wave, Tags) +- Nested cards show all EC2 instances using this Security Group + +### View Security Group Rules + +When viewing security groups (either attached to an EC2 or directly): +1. Click the **View Rules** button on any security group card +2. A modal opens showing all ingress and egress rules +3. Switch between **Ingress** and **Egress** tabs +4. Use the search box to filter rules by protocol, port, source, or description +5. Rules are displayed in a compact table format with: + - Protocol (TCP, UDP, ICMP, All) + - Port Range + - Source Type (CIDR, Security Group, Prefix List) + - Source (IP range or SG ID) + - Description + +### Navigate + +- Click **← Back to Search** to return to search results +- Perform a new search at any time +- Click outside the rules modal to close it + +### Export to CSV + +SGO provides comprehensive CSV export capabilities: + +**Search Results Export:** +- Click the **💾 Export** button in the view controls (top right) +- Exports all current search results with filters applied +- Includes: Type, Name, ID, Account, State, IP, Security Groups count, Wave, Git info + +**EC2 Instance Details Export:** +- Click the **💾 Export** button in any EC2 detail card +- Exports complete EC2 information including: + - Instance details (ID, name, state, IP, account info) + - All AWS tags + - Attached security groups with their details + +**Security Group Details Export:** +- Click the **💾 Export** button in any SG detail card +- Exports complete SG information including: + - Group details (ID, name, wave, rule counts) + - All AWS tags + - Attached EC2 instances with their details + +**Security Group Rules Export:** +- Click the **💾 Export** button in the rules modal +- Exports all ingress and egress rules with: + - Rule details (direction, protocol, ports, source) + - Group ID, account ID + - Git file and commit information from tags + +All exports include timestamps in filenames and proper CSV escaping. + +## Data Structure + +### Security Groups Table +- Account ID & Name +- Group ID & Name +- Tag Name +- Wave Tag +- Git Repo Tag +- Ingress Rule Count + +### EC2 Instances Table +- Account ID & Name +- Instance ID +- Tag Name +- State (running, stopped, etc.) +- Private IP Address +- Security Groups (IDs and Names) +- Git Repo Tag ## File Structure diff --git a/app.py b/app.py index b7b2e99..9391cdf 100755 --- a/app.py +++ b/app.py @@ -140,14 +140,22 @@ def get_profiles(): profiles = [] for section in config.sections(): + profile_name = None if section.startswith('profile '): profile_name = section.replace('profile ', '') - profiles.append(profile_name) elif section == 'default': - profiles.append('default') + profile_name = 'default' + + if profile_name: + # Check if profile has MFA configured + has_mfa = config.has_option(section, 'mfa_serial') + profiles.append({ + 'name': profile_name, + 'has_mfa': has_mfa + }) # Sort profiles alphabetically, but keep 'default' at the top - profiles.sort(key=lambda x: ('0' if x == 'default' else '1' + x.lower())) + profiles.sort(key=lambda x: ('0' if x['name'] == 'default' else '1' + x['name'].lower())) return jsonify({'profiles': profiles}) except Exception as e: diff --git a/templates/import.html b/templates/import.html index bbb970c..b5fbc0e 100644 --- a/templates/import.html +++ b/templates/import.html @@ -190,7 +190,7 @@

SG Observatory

-

Select AWS profiles to import EC2 instances and Security Groups

+

Select AWS profiles to import EC2 instances and Security Groups • Source Code

@@ -202,9 +202,9 @@
-

MFA Codes

+

Import Profiles

- Enter MFA/OTP codes for profiles that require authentication + Enter MFA/TOTP codes where required and start imports for selected profiles

@@ -254,9 +254,10 @@
`).join(''); @@ -274,7 +275,10 @@ selectedProfiles.clear(); document.querySelectorAll('.profile-item input[type="checkbox"]:checked').forEach(cb => { - selectedProfiles.add(cb.value); + selectedProfiles.add({ + name: cb.value, + has_mfa: cb.dataset.hasMfa === 'true' + }); }); if (selectedProfiles.size > 0) { @@ -293,15 +297,15 @@ const savedButtonStates = {}; selectedProfiles.forEach(profile => { - const input = document.getElementById(`mfa-${profile}`); - const btn = document.getElementById(`btn-${profile}`); + const input = document.getElementById(`mfa-${profile.name}`); + const btn = document.getElementById(`btn-${profile.name}`); if (input) { - savedMfaValues[profile] = input.value; + savedMfaValues[profile.name] = input.value; } if (btn) { - savedButtonStates[profile] = { + savedButtonStates[profile.name] = { text: btn.textContent, disabled: btn.disabled, classes: btn.className @@ -309,19 +313,21 @@ } }); - // Render inputs + // Render all selected profiles, but only show MFA input for profiles that have MFA container.innerHTML = Array.from(selectedProfiles).map(profile => `
- +
+ ${profile.has_mfa ? ` + ` : ''}
@@ -330,17 +336,17 @@ // Restore saved values and button states selectedProfiles.forEach(profile => { - const input = document.getElementById(`mfa-${profile}`); - const btn = document.getElementById(`btn-${profile}`); + const input = document.getElementById(`mfa-${profile.name}`); + const btn = document.getElementById(`btn-${profile.name}`); - if (savedMfaValues[profile] !== undefined) { - input.value = savedMfaValues[profile]; + if (savedMfaValues[profile.name] !== undefined) { + input.value = savedMfaValues[profile.name]; } - if (savedButtonStates[profile]) { - btn.textContent = savedButtonStates[profile].text; - btn.disabled = savedButtonStates[profile].disabled; - btn.className = savedButtonStates[profile].classes; + if (savedButtonStates[profile.name]) { + btn.textContent = savedButtonStates[profile.name].text; + btn.disabled = savedButtonStates[profile.name].disabled; + btn.className = savedButtonStates[profile.name].classes; } }); } @@ -356,8 +362,8 @@ btn.textContent = 'Importing...'; progressSection.classList.add('active'); - // Get MFA code for this profile - const mfaCode = mfaInput.value.trim(); + // Get MFA code for this profile (if MFA input exists) + const mfaCode = mfaInput ? mfaInput.value.trim() : ''; try { const response = await fetch('/api/import-profile', { @@ -428,5 +434,13 @@ // Load profiles on page load loadProfiles(); + + diff --git a/templates/index.html b/templates/index.html index 0654a12..5c344ef 100644 --- a/templates/index.html +++ b/templates/index.html @@ -13,7 +13,7 @@

🔭 SGO: Security Groups (and Instances) Observatory

-

Search and explore EC2 instances and Security Groups

+

Search and explore EC2 instances and Security Groups • Source Code