Commit a07be335 authored by Elton Vivot Dias's avatar Elton Vivot Dias
Browse files

added shib files

parent a592da1b
Pipeline #24053 failed with stages
# ------ MONGODB CONFIG ------
MONGODB_HOST=backend-db # docker host
MONGODB_PORT=27017 # docker port
MONGODB_USER=portal
MONGODB_PASSWORD=portal
MONGODB_DATABASE=backend-db
# ------ JWT CONFIG ------
JWT_SECRET_KEY=secret_key
JWT_ALGORITHM=HS256
JWT_EXPIRATION_DAYS=30
JWT_EXPIRATION_SECONDS=0
# ------ APP CONFIG ------
LOG_LEVEL=debug
ROOT_LOGIN=root@portalservice.com
ROOT_PASSWORD=root_pass
RESERVATION_MINIMUM_TIME_IN_MINUTES=30
RESERVATION_MAXIMUM_TIME_IN_HOURS=4320
AUTHORIZED_SERVICES=stackstorm
# ------ SHIB CONFIG ------
SHIB_VOL=./shib/shibboleth
APACHE_VOL=./shib/apache2
LOG_VOL=./shib/logs
APP_PORT=8000
\ No newline at end of file
SERVICE_DNS=dev-cloudnext.cafeexpresso.rnp.br
TIME_ZONE=America/Sao_Paulo
# My files
app/.env
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
stages:
- sonarqube-check
- tests
- build
sonarqube-check:
stage: sonarqube-check
image:
name: sonarsource/sonar-scanner-cli:latest
entrypoint: [""]
variables:
SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar"
GIT_DEPTH: "0"
cache:
key: "${CI_JOB_NAME}"
paths:
- .sonar/cache
script:
- echo "Starting Analyze"
- echo "$SONAR_HOST_URL"
- sonar-scanner -Dsonar.projectKey=DPD_$CI_PROJECT_PATH_SLUG -Dsonar.junit.reportPaths=build/test-results/test/**/TEST-*.xml -Dsonar.coverage.jacoco.xmlReportPaths=build/reports/jacoco/test/jacocoTestReport.xml
- echo "Backend Artifacts Successfully Analyzed"
allow_failure: true
only:
- dev
tests:
stage: tests
variables:
DOCKER_HOST: tcp://docker:2375
DOCKER_TLS_CERTDIR: ""
tags:
- docker
services:
- docker:19.03-dind
script:
- docker network create cloudnext-net --subnet 10.10.10.0/24 --gateway 10.10.10.1
- mkdir dbdata/
- docker run --name cloudnext-db --env MONGO_INITDB_ROOT_USERNAME=orm --env MONGO_INITDB_ROOT_PASSWORD=orm -p 27017:27017 -v dbdata:/data/db --net cloudnext-net --ip 10.10.10.2 -d mongo
- docker build -t cloudnext-app:latest .
- docker run --name cloudnext-app -p 5000:5000 --env-file test/.env.txt --net cloudnext-net --ip 10.10.10.3 -d cloudnext-app:latest
- sleep 10
- docker exec cloudnext-app python3 test/populate_test_objects.py
- docker exec cloudnext-app python3 -m pytest test/* -v
only:
- dev
build:
stage: build
variables:
DOCKER_HOST: tcp://docker:2375
DOCKER_TLS_CERTDIR: ""
tags:
- docker
services:
- docker:19.03-dind
script:
- docker build -t cloudnext-app:latest .
- docker login http://nexus.ids.rnp.br:8085 -u $DPDI_NEXUS_USER -p $DPDI_NEXUS_PASSWD
- docker tag docker tag cloudnext-app nexus.ids.rnp.br:8085/cloudnext-app:latest
- docker push nexus.ids.rnp.br:8085/cloudnext-app:latest
only:
- dev
\ No newline at end of file
FROM python:3.6-slim-buster
WORKDIR /app
# install dependencies
RUN apt-get update && apt-get install gcc -y && apt-get clean
# install requirements
COPY ./app/requirements.txt requirements.txt
RUN pip3 install -r requirements.txt
# copy app files
COPY ./app .
# copy test files except populate_test_objects.py
COPY ./test ./test
# CMD [ "/usr/local/bin/uwsgi", "--protocol=http", "uwsgi-server.ini"]
CMD [ "/usr/local/bin/uwsgi", "uwsgi-server.ini"]
# python-orm
Testing python ORMs and DBs
## Setup configuration
create .env file.
Structure:
```
# ------ MONGODB CONFIG ------
MONGODB_HOST= # docker host
MONGODB_PORT= # docker port
MONGODB_USER=
MONGODB_PASSWORD=
MONGODB_DATABASE=
# ------ JWT CONFIG ------
JWT_SECRET_KEY=secret_key
JWT_ALGORITHM=HS256
JWT_EXPIRATION_DAYS=30
JWT_EXPIRATION_SECONDS=0
# ------ APP CONFIG ------
LOG_LEVEL=debug
ROOT_LOGIN=insert-your-root-user-here@portal.com
ROOT_PASSWORD=insert-your-root-password-here
RESERVATION_MINIMUM_TIME_IN_MINUTES=30
RESERVATION_MAXIMUM_TIME_IN_HOURS=4320
AUTHORIZED_SERVICES=test0,test1
```
## start app
```
docker-compose up -d
```
## logging
```
docker logs -f orm-app
```
## development
First config:
```
python3 -m venv env
source env/bin/activate
pip3 install -r ./app/requirements.txt
```
Update ´requirements.tx´:
```
source env/bin/activate
pip3 install package
pip3 freeze > requirements.txt
```
Update docker image:
```
docker-compose up --build -d
```
Import logger in module (file):
```
import logging
logger = logging.getLogger(__name__)
```
from app.config import log_level
import logging, sys
print(sys.path)
# init logs
logger = logging.getLogger(__name__)
f='%(asctime)s | %(levelname)s | %(name)s:%(funcName)s | %(message)s'
if log_level != logging.DEBUG: f='%(asctime)s | %(levelname)s | %(message)s'
logging.basicConfig(level=log_level, format=f)
\ No newline at end of file
from flask import Flask
from flask_cors import CORS
from dotenv import load_dotenv
import logging,sys
def create_app():
load_dotenv()
import models, rest
app = Flask(__name__)
CORS(app, resources={r"/*": {"origins": "*"}})
models.init_app(app)
rest.init_app(app)
return app
if __name__ == '__main__':
app = create_app()
app.run()
import sys, argparse, datetime, jwt
sys.path.insert(0,'../')
from app.config import jwt_secret_key, jwt_algorithm
def encode_auth_token(data,secret_key=jwt_secret_key):
try:
payload = {
'exp': datetime.datetime.utcnow() + datetime.timedelta(days=365,seconds=0),
'iat' : datetime.datetime.utcnow(),
'data' : data
}
return jwt.encode(payload, secret_key, algorithm=jwt_algorithm)
except Exception as err:
print(f"{err.__class__.__name__}: {err}")
raise
def main():
parser = argparse.ArgumentParser(description="Backend functions")
group = parser.add_mutually_exclusive_group()
group.add_argument("-ct", "--create-token", type=str, help="Create member token with provided string value")
args = parser.parse_args()
if args.create_token:
from app.controller.auth import encode_auth_token
token_data = {'member_id': str(args.create_token)}
auth_token = encode_auth_token(data=token_data)
print(f"\nGenerated token: \n{auth_token}")
if __name__ == '__main__':
main()
\ No newline at end of file
import os, logging
# db config
db_host = str(os.getenv("MONGODB_HOST"))
db_port = str(os.getenv("MONGODB_PORT"))
db_user = str(os.getenv("MONGODB_USER"))
db_pass = str(os.getenv("MONGODB_PASSWORD"))
db_dbase = str(os.getenv("MONGODB_DATABASE"))
# jwt config
jwt_secret_key = str(os.getenv("JWT_SECRET_KEY"))
jwt_algorithm = str(os.getenv("JWT_ALGORITHM"))
jwt_exp_days = int(str(os.getenv("JWT_EXPIRATION_DAYS")))
jwt_exp_seconds = int(str(os.getenv("JWT_EXPIRATION_SECONDS")))
# app config
_log_levels = {
'debug': logging.DEBUG,
'info': logging.INFO,
'warning': logging.WARNING,
'error': logging.ERROR,
'critical': logging.CRITICAL,
'': logging.NOTSET
}
log_level = _log_levels.get(str(os.getenv("LOG_LEVEL")))
root_login = str(os.getenv("ROOT_LOGIN"))
root_pwd = str(os.getenv("ROOT_PASSWORD"))
res_min_time = int(str(os.getenv("RESERVATION_MINIMUM_TIME_IN_MINUTES"))) # minutes
res_max_time = int(str(os.getenv("RESERVATION_MAXIMUM_TIME_IN_HOURS"))) # hours
authorized_services = str(os.getenv("AUTHORIZED_SERVICES")).split(",")
from app.config import jwt_secret_key, jwt_algorithm, jwt_exp_days, jwt_exp_seconds
import jwt, logging, datetime
logger = logging.getLogger(__name__)
def encode_auth_token(data,secret_key=jwt_secret_key):
try:
payload = {
'exp': datetime.datetime.utcnow() + datetime.timedelta(days=jwt_exp_days,seconds=jwt_exp_seconds),
'iat' : datetime.datetime.utcnow(),
'data' : data
}
return jwt.encode(payload, secret_key, algorithm=jwt_algorithm)
except Exception as err:
logger.error(f"{err.__class__.__name__}: {err}")
raise
def decode_auth_token(auth_token,secret_key=jwt_secret_key):
try:
payload = jwt.decode(auth_token,secret_key, algorithms=jwt_algorithm)
return payload['data']
except Exception as err:
logger.error(f"{err.__class__.__name__}: {err}")
raise
\ No newline at end of file
from app.rest.errors import NotFoundError
import sys, logging, json
logger = logging.getLogger(__name__)
def parse_type(data_type, module):
try:
return getattr(sys.modules[module], data_type)
except Exception:
raise NotFoundError("Path not found.")
def get_all(data_type, module):
try:
obj_type = parse_type(data_type, module)
method = getattr(obj_type, "get_all", None)
if callable(method): return method()
else: return obj_type.objects()
except Exception as err:
logger.error(f"{err.__class__.__name__}: {err}")
raise
def get(data_id, data_type, module):
try:
obj_type = parse_type(data_type, module)
method = getattr(obj_type, "get", None)
if callable(method): return method(data_id)
else: return obj_type.objects(id=data_id).first()
except Exception as err:
logger.error(f"{err.__class__.__name__}: {err}")
raise
def insert(json_data, data_type, module):
try:
obj_type = parse_type(data_type, module)
new_data = obj_type(**json_data)
new_data.save()
return new_data
except Exception as err:
logger.error(f"{err.__class__.__name__}: {err}")
raise
def update(json_data, data_id, data_type, module):
try:
data = get(data_id, data_type, module)
data.from_json(json.dumps(json_data), created=True)
data.save()
return data
except Exception as err:
logger.error(f"{err.__class__.__name__}: {err}")
raise
def delete(data_id, data_type, module):
try:
data = get(data_id, data_type, module)
data.delete()
except Exception as err:
logger.error(f"{err.__class__.__name__}: {err}")
raise
\ No newline at end of file
from mongoengine import connect, NotUniqueError
# from app.config import db_user, db_pass, db_host, db_port, db_dbase, root_login, root_pwd
# from app import config
from app.config import db_user, db_pass, db_host, db_port, db_dbase, root_login, root_pwd
import logging
logger = logging.getLogger(__name__)
def init_app(app):
# database config
connect(
db=db_dbase,
host=f"mongodb://{db_user}:{db_pass}@{db_host}:{db_port}/?authSource=admin"
)
populate_db()
def populate_db():
from app.models.member import Member
try:
ssh_keys = Member.generate_ssh_keys()
root = Member(email=root_login, password=Member.hash_string(root_pwd), privilege=4, ssh_keys=ssh_keys)
root.save()
if not Member.objects(email=root_login): logger.error("Failed to create root user")
else: logger.info("Created root user")
except NotUniqueError:
logger.info("Created root user")
pass
except Exception as err:
logger.debug(f"db_user: {db_user}\ndb_pass: {db_pass}\ndb_host: {db_host}\ndb_port: {db_port}\ndb_dbase: {db_dbase}\nroot_login: {root_login}\nroot_pwd: {root_pwd}")
logger.error(f"{err.__class__.__name__}: {err}")
raise
from mongoengine_goodjson import Document
from mongoengine import StringField, ListField, MapField, BooleanField, ReferenceField
from app.rest.errors import NotFoundError
class Image(Document):
name = StringField()
attributes = MapField(field=StringField())
# compatibles = ListField(ReferenceField(ResourceType)) #v1+
@staticmethod
def get(id):
image = Image.objects(id=id).first()
if not image: raise NotFoundError(f"Image with id = '{id}'not found")
return image
\ No newline at end of file
from mongoengine_goodjson import Document, EmbeddedDocument
from mongoengine import signals, StringField, BooleanField, IntField, EmbeddedDocumentField
from cryptography.hazmat.primitives import serialization as crypto_serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.backends import default_backend as crypto_default_backend
from app.rest.errors import NotFoundError, BadRequestError, NotAuthorizedError
import bcrypt, json, sys
class SSHKeys(EmbeddedDocument):
private_key = StringField()
public_key = StringField()
class Member(Document):
first_name = StringField()
last_name = StringField()
affiliation = StringField()
email = StringField(unique=True)
password = StringField()
country = StringField()
city = StringField()
privilege = IntField(default=1)
enabled = BooleanField(default=True)
is_federated = BooleanField(default=False)
ssh_keys = EmbeddedDocumentField(SSHKeys)
@classmethod
def post_save(cls, sender, document, **kwargs):
document.password = None
@staticmethod
def get_all():
return Member.objects(privilege__lt=4).exclude('password').exclude('ssh_keys')
@staticmethod
def get(id):
member = Member.objects(id=id).exclude('password').exclude('ssh_keys').first()
if not member: raise NotFoundError(f"Member with id = '{id}'not found")
return member
@staticmethod
def get_ssh_keys(id):
member = Member.objects(id=id).only('ssh_keys').first()
if not member: raise NotFoundError(f"Member with id = '{id}'not found")
return member
@staticmethod
def hash_string(string):
return (bcrypt.hashpw(string.encode('utf-8'), bcrypt.gensalt())).decode('utf-8')
@staticmethod
def validate_create(member_dict):
if not 'password' in member_dict: raise BadRequestError("You must create a password (min 8 characters)")
if len(member_dict['password']) < 8: raise BadRequestError("Password must have at least 8 characters")
member_dict['password'] = Member.hash_string(member_dict['password'])
member_dict['privilege'] = 1
member_dict['is_federated'] = False
member_dict['ssh_keys'] = json.loads(Member.generate_ssh_keys().to_json())
return member_dict
@staticmethod
def validate_update(member_dict):
if 'password' in member_dict:
if len(member_dict['password']) < 8: raise BadRequestError("Password must have at least 8 characters")
member_dict['password'] = Member.hash_string(member_dict['password'])