diff --git a/.gitignore b/.gitignore index 56f1350..6174009 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ data*/ *.yaml !config.yaml kvs +*.log +extract_*.py diff --git a/config/config.go b/config/config.go index edc4b2a..1eed905 100644 --- a/config/config.go +++ b/config/config.go @@ -28,22 +28,22 @@ func Default() *types.Config { ThrottleDelayMs: 100, FetchDelayMs: 50, - // Phase 2: Default compression settings + // Default compression settings CompressionEnabled: true, CompressionLevel: 3, // Balance between performance and compression ratio - // Phase 2: Default TTL and size limit settings + // Default TTL and size limit settings DefaultTTL: "0", // No default TTL MaxJSONSize: 1048576, // 1MB default max JSON size - // Phase 2: Default rate limiting settings + // Default rate limiting settings RateLimitRequests: 100, // 100 requests per window RateLimitWindow: "1m", // 1 minute window - // Phase 2: Default tamper-evident logging settings + // Default tamper-evident logging settings TamperLogActions: []string{"data_write", "user_create", "auth_failure"}, - // Phase 2: Default backup system settings + // Default backup system settings BackupEnabled: true, BackupSchedule: "0 0 * * *", // Daily at midnight BackupPath: "./backups", diff --git a/features/auth.go b/features/auth.go new file mode 100644 index 0000000..65580a5 --- /dev/null +++ b/features/auth.go @@ -0,0 +1,102 @@ +package features + +import ( + "fmt" + "net/http" + "strings" + + "github.com/gorilla/mux" + + "kvs/types" +) + +// AuthContext holds authentication information for a request +type AuthContext struct { + UserUUID string `json:"user_uuid"` + Scopes []string `json:"scopes"` + Groups []string `json:"groups"` +} + +// CheckPermission validates if a user has permission to perform an operation +func CheckPermission(permissions int, operation string, isOwner, isGroupMember bool) bool { + switch operation { + case "create": + if isOwner { + return (permissions & types.PermOwnerCreate) != 0 + } + if isGroupMember { + return (permissions & types.PermGroupCreate) != 0 + } + return (permissions & types.PermOthersCreate) != 0 + + case "delete": + if isOwner { + return (permissions & types.PermOwnerDelete) != 0 + } + if isGroupMember { + return (permissions & types.PermGroupDelete) != 0 + } + return (permissions & types.PermOthersDelete) != 0 + + case "write": + if isOwner { + return (permissions & types.PermOwnerWrite) != 0 + } + if isGroupMember { + return (permissions & types.PermGroupWrite) != 0 + } + return (permissions & types.PermOthersWrite) != 0 + + case "read": + if isOwner { + return (permissions & types.PermOwnerRead) != 0 + } + if isGroupMember { + return (permissions & types.PermGroupRead) != 0 + } + return (permissions & types.PermOthersRead) != 0 + + default: + return false + } +} + +// CheckUserResourceRelationship determines user relationship to resource +func CheckUserResourceRelationship(userUUID string, metadata *types.ResourceMetadata, userGroups []string) (isOwner, isGroupMember bool) { + isOwner = (userUUID == metadata.OwnerUUID) + + if metadata.GroupUUID != "" { + for _, groupUUID := range userGroups { + if groupUUID == metadata.GroupUUID { + isGroupMember = true + break + } + } + } + + return isOwner, isGroupMember +} + +// ExtractTokenFromHeader extracts the Bearer token from the Authorization header +func ExtractTokenFromHeader(r *http.Request) (string, error) { + authHeader := r.Header.Get("Authorization") + if authHeader == "" { + return "", fmt.Errorf("missing authorization header") + } + + parts := strings.Split(authHeader, " ") + if len(parts) != 2 || strings.ToLower(parts[0]) != "bearer" { + return "", fmt.Errorf("invalid authorization header format") + } + + return parts[1], nil +} + +// ExtractKVResourceKey extracts KV resource key from request +func ExtractKVResourceKey(r *http.Request) string { + vars := mux.Vars(r) + if path, ok := vars["path"]; ok { + return path + } + return "" +} \ No newline at end of file diff --git a/features/backup.go b/features/backup.go new file mode 100644 index 0000000..893351b --- /dev/null +++ b/features/backup.go @@ -0,0 +1,11 @@ +package features + +import ( + "fmt" + "time" +) + +// GetBackupFilename generates a filename for a backup +func GetBackupFilename(timestamp time.Time) string { + return fmt.Sprintf("kvs-backup-%s.zstd", timestamp.Format("2006-01-02")) +} \ No newline at end of file diff --git a/features/features.go b/features/features.go new file mode 100644 index 0000000..6ee027e --- /dev/null +++ b/features/features.go @@ -0,0 +1,4 @@ +// Package features provides utility functions for KVS authentication, validation, +// logging, backup, and other operational features. These functions were extracted +// from main.go to improve code organization and maintainability. +package features \ No newline at end of file diff --git a/features/ratelimit.go b/features/ratelimit.go new file mode 100644 index 0000000..99ef49b --- /dev/null +++ b/features/ratelimit.go @@ -0,0 +1,8 @@ +package features + +import "fmt" + +// GetRateLimitKey generates the storage key for rate limiting +func GetRateLimitKey(userUUID string, windowStart int64) string { + return fmt.Sprintf("ratelimit:%s:%d", userUUID, windowStart) +} \ No newline at end of file diff --git a/features/revision.go b/features/revision.go new file mode 100644 index 0000000..d348010 --- /dev/null +++ b/features/revision.go @@ -0,0 +1,8 @@ +package features + +import "fmt" + +// GetRevisionKey generates the storage key for a specific revision +func GetRevisionKey(baseKey string, revision int) string { + return fmt.Sprintf("%s:rev:%d", baseKey, revision) +} \ No newline at end of file diff --git a/features/tamperlog.go b/features/tamperlog.go new file mode 100644 index 0000000..6e57f21 --- /dev/null +++ b/features/tamperlog.go @@ -0,0 +1,24 @@ +package features + +import ( + "fmt" + + "kvs/utils" +) + +// GetTamperLogKey generates the storage key for a tamper log entry +func GetTamperLogKey(timestamp string, entryUUID string) string { + return fmt.Sprintf("log:%s:%s", timestamp, entryUUID) +} + +// GetMerkleLogKey generates the storage key for hourly Merkle tree roots +func GetMerkleLogKey(timestamp string) string { + return fmt.Sprintf("log:merkle:%s", timestamp) +} + +// GenerateLogSignature creates a SHA3-512 signature for a log entry +func GenerateLogSignature(timestamp, action, userUUID, resource string) string { + // Concatenate all fields in a deterministic order + data := fmt.Sprintf("%s|%s|%s|%s", timestamp, action, userUUID, resource) + return utils.HashSHA3512(data) +} \ No newline at end of file diff --git a/features/validation.go b/features/validation.go new file mode 100644 index 0000000..096c3b5 --- /dev/null +++ b/features/validation.go @@ -0,0 +1,24 @@ +package features + +import ( + "fmt" + "time" +) + +// ParseTTL converts a Go duration string to time.Duration +func ParseTTL(ttlString string) (time.Duration, error) { + if ttlString == "" || ttlString == "0" { + return 0, nil // No TTL + } + + duration, err := time.ParseDuration(ttlString) + if err != nil { + return 0, fmt.Errorf("invalid TTL format: %v", err) + } + + if duration < 0 { + return 0, fmt.Errorf("TTL cannot be negative") + } + + return duration, nil +} \ No newline at end of file diff --git a/main.go b/main.go index 5190f3f..ebcc2d9 100644 --- a/main.go +++ b/main.go @@ -5,284 +5,12 @@ import ( "net/http" "os" "os/signal" - "strings" "syscall" - "time" - - "github.com/golang-jwt/jwt/v4" - "github.com/gorilla/mux" "kvs/config" "kvs/server" - "kvs/types" - "kvs/utils" ) -// Phase 2: Permission checking utilities -func checkPermission(permissions int, operation string, isOwner, isGroupMember bool) bool { - switch operation { - case "create": - if isOwner { - return (permissions & types.PermOwnerCreate) != 0 - } - if isGroupMember { - return (permissions & types.PermGroupCreate) != 0 - } - return (permissions & types.PermOthersCreate) != 0 - - case "delete": - if isOwner { - return (permissions & types.PermOwnerDelete) != 0 - } - if isGroupMember { - return (permissions & types.PermGroupDelete) != 0 - } - return (permissions & types.PermOthersDelete) != 0 - - case "write": - if isOwner { - return (permissions & types.PermOwnerWrite) != 0 - } - if isGroupMember { - return (permissions & types.PermGroupWrite) != 0 - } - return (permissions & types.PermOthersWrite) != 0 - - case "read": - if isOwner { - return (permissions & types.PermOwnerRead) != 0 - } - if isGroupMember { - return (permissions & types.PermGroupRead) != 0 - } - return (permissions & types.PermOthersRead) != 0 - - default: - return false - } -} - -// Helper function to determine user relationship to resource -func checkUserResourceRelationship(userUUID string, metadata *types.ResourceMetadata, userGroups []string) (isOwner, isGroupMember bool) { - isOwner = (userUUID == metadata.OwnerUUID) - - if metadata.GroupUUID != "" { - for _, groupUUID := range userGroups { - if groupUUID == metadata.GroupUUID { - isGroupMember = true - break - } - } - } - - return isOwner, isGroupMember -} - -// Phase 2: JWT token management utilities - -// JWT signing key (should be configurable in production) -var jwtSigningKey = []byte("your-secret-signing-key-change-this-in-production") - -// JWTClaims represents the custom claims for our JWT tokens -type JWTClaims struct { - UserUUID string `json:"user_uuid"` - Scopes []string `json:"scopes"` - jwt.RegisteredClaims -} - -// generateJWT creates a new JWT token for a user with specified scopes - -// validateJWT validates a JWT token and returns the claims if valid -func validateJWT(tokenString string) (*JWTClaims, error) { - token, err := jwt.ParseWithClaims(tokenString, &JWTClaims{}, func(token *jwt.Token) (interface{}, error) { - // Validate signing method - if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { - return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) - } - return jwtSigningKey, nil - }) - - if err != nil { - return nil, err - } - - if claims, ok := token.Claims.(*JWTClaims); ok && token.Valid { - return claims, nil - } - - return nil, fmt.Errorf("invalid token") -} - -// storeAPIToken stores an API token in BadgerDB with TTL - -// getAPIToken retrieves an API token from BadgerDB by hash - -// Phase 2: Authorization middleware - -// AuthContext holds authentication information for a request -type AuthContext struct { - UserUUID string `json:"user_uuid"` - Scopes []string `json:"scopes"` - Groups []string `json:"groups"` -} - -// extractTokenFromHeader extracts the Bearer token from the Authorization header -func extractTokenFromHeader(r *http.Request) (string, error) { - authHeader := r.Header.Get("Authorization") - if authHeader == "" { - return "", fmt.Errorf("missing authorization header") - } - - parts := strings.Split(authHeader, " ") - if len(parts) != 2 || strings.ToLower(parts[0]) != "bearer" { - return "", fmt.Errorf("invalid authorization header format") - } - - return parts[1], nil -} - -// getUserGroups retrieves all groups that a user belongs to - -// authenticateRequest validates the JWT token and returns authentication context - -// checkResourcePermission checks if a user has permission to perform an operation on a resource - -// authMiddleware is the HTTP middleware that enforces authentication and authorization - -// Helper function to extract KV resource key from request -func extractKVResourceKey(r *http.Request) string { - vars := mux.Vars(r) - if path, ok := vars["path"]; ok { - return path - } - return "" -} - -// Phase 2: ZSTD compression utilities - -// compressData compresses JSON data using storage service - -// decompressData decompresses data using storage service - -// Phase 2: TTL and size validation utilities - -// parseTTL converts a Go duration string to time.Duration -func parseTTL(ttlString string) (time.Duration, error) { - if ttlString == "" || ttlString == "0" { - return 0, nil // No TTL - } - - duration, err := time.ParseDuration(ttlString) - if err != nil { - return 0, fmt.Errorf("invalid TTL format: %v", err) - } - - if duration < 0 { - return 0, fmt.Errorf("TTL cannot be negative") - } - - return duration, nil -} - -// validateJSONSize checks if JSON data exceeds maximum allowed size - -// createResourceMetadata creates metadata for a new resource with TTL and permissions - -// storeWithTTL stores data in BadgerDB with optional TTL - -// retrieveWithDecompression retrieves and decompresses data from BadgerDB - -// Phase 2: Revision history system utilities - -// getRevisionKey generates the storage key for a specific revision -func getRevisionKey(baseKey string, revision int) string { - return fmt.Sprintf("%s:rev:%d", baseKey, revision) -} - -// storeRevisionHistory stores a value and manages revision history (up to 3 revisions) -func getRateLimitKey(userUUID string, windowStart int64) string { - return fmt.Sprintf("ratelimit:%s:%d", userUUID, windowStart) -} - -// getCurrentWindow calculates the current rate limiting window start time - -// checkRateLimit checks if a user has exceeded the rate limit - -// rateLimitMiddleware is the HTTP middleware that enforces rate limiting - -// Phase 2: Tamper-evident logging utilities - -// getTamperLogKey generates the storage key for a tamper log entry -func getTamperLogKey(timestamp string, entryUUID string) string { - return fmt.Sprintf("log:%s:%s", timestamp, entryUUID) -} - -// getMerkleLogKey generates the storage key for hourly Merkle tree roots -func getMerkleLogKey(timestamp string) string { - return fmt.Sprintf("log:merkle:%s", timestamp) -} - -// generateLogSignature creates a SHA3-512 signature for a log entry -func generateLogSignature(timestamp, action, userUUID, resource string) string { - // Concatenate all fields in a deterministic order - data := fmt.Sprintf("%s|%s|%s|%s", timestamp, action, userUUID, resource) - return utils.HashSHA3512(data) -} - -// isActionLogged checks if a specific action should be logged - -// createTamperLogEntry creates a new tamper-evident log entry - -// storeTamperLogEntry stores a tamper-evident log entry in BadgerDB - -// logTamperEvent logs a tamper-evident event if the action is configured for logging - -// getTamperLogs retrieves tamper log entries within a time range (for auditing) - -// Phase 2: Backup system utilities - -// getBackupFilename generates a filename for a backup -func getBackupFilename(timestamp time.Time) string { - return fmt.Sprintf("kvs-backup-%s.zstd", timestamp.Format("2006-01-02")) -} - -// createBackup creates a compressed backup of the BadgerDB database - -// cleanupOldBackups removes backup files older than the retention period - -// initializeBackupScheduler sets up the cron scheduler for automated backups - -// getBackupStatus returns the current backup status - -// Initialize server - -// performMerkleSync performs a synchronization round using Merkle Trees - -// requestMerkleRoot requests the Merkle root from a peer - -// diffMerkleTreesRecursive recursively compares local and remote Merkle tree nodes - -// requestMerkleDiff requests children hashes or keys for a given node/range from a peer - -// fetchSingleKVFromPeer fetches a single KV pair from a peer - -// storeReplicatedDataWithMetadata stores replicated data preserving its original metadata - -// deleteKVLocally deletes a key-value pair and its associated timestamp index locally. - -// fetchAndStoreRange fetches a range of KV pairs from a peer and stores them locally - -// Bootstrap - join cluster using seed nodes - -// Attempt to join cluster via a seed node - -// Perform gradual sync (Merkle-based version) - -// Resolve conflict between local and remote data using majority vote and oldest node tie-breaker - -// Resolve conflict using oldest node rule when no other members available - -// getLocalData is a utility to retrieve a types.StoredValue from local DB. func main() { configPath := "./config.yaml" diff --git a/server/handlers.go b/server/handlers.go index d750063..843f530 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -13,7 +13,6 @@ import ( "time" "github.com/dgraph-io/badger/v4" - "github.com/golang-jwt/jwt/v4" "github.com/google/uuid" "github.com/gorilla/mux" "github.com/sirupsen/logrus" @@ -23,14 +22,7 @@ import ( "kvs/utils" ) -// JWTClaims represents the custom claims for JWT tokens -type JWTClaims struct { - UserUUID string `json:"user_uuid"` - Scopes []string `json:"scopes"` - jwt.RegisteredClaims -} -var jwtSigningKey = []byte("your-super-secret-key") // TODO: Move to config // healthHandler returns server health status func (s *Server) healthHandler(w http.ResponseWriter, r *http.Request) { @@ -998,7 +990,7 @@ func (s *Server) createTokenHandler(w http.ResponseWriter, r *http.Request) { } // Generate JWT token - tokenString, expiresAt, err := generateJWT(req.UserUUID, req.Scopes, 1) // 1 hour default + tokenString, expiresAt, err := auth.GenerateJWT(req.UserUUID, req.Scopes, 1) // 1 hour default if err != nil { s.logger.WithError(err).Error("Failed to generate JWT token") http.Error(w, "Internal Server Error", http.StatusInternalServerError) @@ -1241,33 +1233,6 @@ func (s *Server) buildMerkleTreeRecursive(nodes []*types.MerkleNode) (*types.Mer return s.buildMerkleTreeRecursive(nextLevel) } -// generateJWT creates a new JWT token for a user with specified scopes -func generateJWT(userUUID string, scopes []string, expirationHours int) (string, int64, error) { - if expirationHours <= 0 { - expirationHours = 1 // Default to 1 hour - } - - now := time.Now() - expiresAt := now.Add(time.Duration(expirationHours) * time.Hour) - - claims := JWTClaims{ - UserUUID: userUUID, - Scopes: scopes, - RegisteredClaims: jwt.RegisteredClaims{ - IssuedAt: jwt.NewNumericDate(now), - ExpiresAt: jwt.NewNumericDate(expiresAt), - Issuer: "kvs-server", - }, - } - - token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - tokenString, err := token.SignedString(jwtSigningKey) - if err != nil { - return "", 0, err - } - - return tokenString, expiresAt.Unix(), nil -} func (s *Server) storeAPIToken(tokenString string, userUUID string, scopes []string, expiresAt int64) error { tokenHash := utils.HashToken(tokenString) diff --git a/server/server.go b/server/server.go index e1632ee..0c04348 100644 --- a/server/server.go +++ b/server/server.go @@ -41,7 +41,7 @@ type Server struct { storageService *storage.StorageService revisionService *storage.RevisionService - // Phase 2: Backup system + // Backup system cronScheduler *cron.Cron // Cron scheduler for backups backupStatus types.BackupStatus // Current backup status backupMu sync.RWMutex // Protects backup status diff --git a/types/types.go b/types/types.go index f725979..c19512d 100644 --- a/types/types.go +++ b/types/types.go @@ -9,7 +9,7 @@ type StoredValue struct { Data json.RawMessage `json:"data"` } -// Phase 2: Authentication & Authorization data structures +// Authentication & Authorization data structures // User represents a system user type User struct { @@ -74,7 +74,7 @@ const ( (PermOthersRead) ) -// Phase 2: API request/response structures for authentication endpoints +// API request/response structures for authentication endpoints // User Management API structures type CreateUserRequest struct { @@ -167,13 +167,13 @@ type PutResponse struct { Timestamp int64 `json:"timestamp"` } -// Phase 2: TTL-enabled PUT request structure +// TTL-enabled PUT request structure type PutWithTTLRequest struct { Data json.RawMessage `json:"data"` TTL string `json:"ttl,omitempty"` // Go duration format } -// Phase 2: Tamper-evident logging data structures +// Tamper-evident logging data structures type TamperLogEntry struct { Timestamp string `json:"timestamp"` // RFC3339 format Action string `json:"action"` // Type of action @@ -182,7 +182,7 @@ type TamperLogEntry struct { Signature string `json:"signature"` // SHA3-512 hash of all fields } -// Phase 2: Backup system data structures +// Backup system data structures type BackupStatus struct { LastBackupTime int64 `json:"last_backup_time"` // Unix timestamp LastBackupSuccess bool `json:"last_backup_success"` // Whether last backup succeeded @@ -246,22 +246,22 @@ type Config struct { ThrottleDelayMs int `yaml:"throttle_delay_ms"` FetchDelayMs int `yaml:"fetch_delay_ms"` - // Phase 2: Database compression configuration + // Database compression configuration CompressionEnabled bool `yaml:"compression_enabled"` CompressionLevel int `yaml:"compression_level"` - // Phase 2: TTL configuration + // TTL configuration DefaultTTL string `yaml:"default_ttl"` // Go duration format, "0" means no default TTL MaxJSONSize int `yaml:"max_json_size"` // Maximum JSON size in bytes - // Phase 2: Rate limiting configuration + // Rate limiting configuration RateLimitRequests int `yaml:"rate_limit_requests"` // Max requests per window RateLimitWindow string `yaml:"rate_limit_window"` // Window duration (Go format) - // Phase 2: Tamper-evident logging configuration + // Tamper-evident logging configuration TamperLogActions []string `yaml:"tamper_log_actions"` // Actions to log - // Phase 2: Backup system configuration + // Backup system configuration BackupEnabled bool `yaml:"backup_enabled"` // Enable/disable automated backups BackupSchedule string `yaml:"backup_schedule"` // Cron schedule format BackupPath string `yaml:"backup_path"` // Directory to store backups diff --git a/utils/hash.go b/utils/hash.go index c354c50..1595b6a 100644 --- a/utils/hash.go +++ b/utils/hash.go @@ -5,7 +5,7 @@ import ( "golang.org/x/crypto/sha3" ) -// SHA3-512 hashing utilities for Phase 2 authentication +// SHA3-512 hashing utilities for authentication func HashSHA3512(input string) string { hasher := sha3.New512() hasher.Write([]byte(input))