Merge pull request #68 from matrix-org/poljar-python

Poljar's improved python bindings

Makefile

@@ -161,6 +161,7 @@ $(RELEASE_TARGET): $(RELEASE_OBJECTS)
             $(OLM_LDFLAGS) \
             $(OUTPUT_OPTION) $(RELEASE_OBJECTS)
 	ln -sf libolm.$(SO).$(VERSION) $(BUILD_DIR)/libolm.$(SO).$(MAJOR)
+	ln -sf libolm.$(SO).$(VERSION) $(BUILD_DIR)/libolm.$(SO)
 
 debug: $(DEBUG_TARGET)
 .PHONY: debug

fuzzers/README.rst

@@ -19,8 +19,7 @@ Usage notes:
       make fuzzers
 
 3. Some of the tests (eg ``fuzz_decrypt`` and ``fuzz_group_decrypt``) require a
-   session file. You can use the ones generated by the python test script
-   (``python/test.sh``).
+   session file. You can create one by pickling an Olm session.
 
 4. Make some work directories:
 

jenkins.sh

@@ -8,14 +8,6 @@ rm -f olm-*.tgz
 make lib
 make test
 
-virtualenv env
-. env/bin/activate
-pip install pyyaml
-pip install pep8
-
-./python/test_olm.sh
-pep8 -v python
-
 . ~/.emsdk_set_env.sh
 make js
 (cd javascript && npm install && npm run test)

python/.gdb_history

-b _olm_enc_input
-r
-l
-p key
-p key_lenght
-p key_length
-b _olm_enc_input
-r
-key[12]
-p key[12]
-p key[11]
-key[11]='\0'
-p key[11]='\0'
-p key[11]
-key_length=12
-p key_length=12
-n
-c
-b _olm_enc_input
-r
-r
-r
-b olm_decrypt
-r
-l
-b 677
-c
-s
-fin
-s
-s
-fin
-s
-s
-fin
-s
-l
-n
-l
-l
-s
-s
-n
-l
-n
-l
-p reader
-p *this
-n
-p chain
-p receiver_chains 
-p receiver_chains.length()
-p receiver_chains.size()
-p reader
-p reader.ratchet_key
-r
-r
-b olm_account_one_time_keys
-r
-l
-s
-n
-p *this
-p one_time_keys
-p one_time_keys.length
-p one_time_keys.length()
-p one_time_keys.len()
-p one_time_keys.size()
-p one_time_keys.count()
-p one_time_keys.data
-p one_time_keys._data
-p &one_time_keys._data
-l
-n
-q
-r
-b olm_create_inbound_session
-r
-b olm_create_inbound_session_from
-r
-r
-r
-b olm_create_inbound_session_from
-r
-b olm_create_inbound_session
-b olm_create_inbound_session
-r
-l
-n
-l
-s
-b olm_create_inbound_session
-r
-l
-l
-n
-s
-f
-s
-fin
-s
-s
-fin
-s
-l
-n
-l
-l -
-l
-l
-l
-n
-p our_one_time_key 
-p *our_one_time_key 
-l
-n
-l
-n
-p bob_one_time_key 
-p alice_identity_key 
-p alice_base_key 
-p bob_identity_key 
-x alice_identity_key 
-x &alice_identity_key 
-x /32x &alice_identity_key 
-x /32b &alice_identity_key 
-l
-l
-l
-n
-b olm_decrypt
-c
-l
-l
-b 'olm::Session::decrypt'
-c
-l
-l
-n
-l
-n
-p reader
-p reader
-5*128
-p 5*128
-p 0xb0 - 0x80
-p 0xb0 - 0x80 + 640
-l
-n
-s
-l
-n
-p reader
-n
-l
-n
-p max_length 
-p reader.ciphertext_length 
-l
-n
-l
-p receiver_chains 
-p &receiver_chains ._data
-p &receiver_chains ._data[1]
-n
-s
-s
-l
-n
-p new_chain.index
-p reader.counter
-n
-l
-l
-n
-s
-s
-n
-l
-x key
-x /16b key
-l
-l
-n
-p keys
-_olm_crypto_aes_decrypt_cbc&keys.aes_key, &keys.aes_iv, ciphertext, ciphertext_length, plaintext
-p _olm_crypto_aes_decrypt_cbc(&keys.aes_key, &keys.aes_iv, ciphertext, ciphertext_length, plaintext)
-p plaintext
-r
-b olm_account_identity_keys
-l
-r
-b olm_unpickle_account
-r
-l
-n
-p object.last_error
-l
-l -
-l
-b 268
-r
-c
-s
-l
-l
-p end-pos
-x /246b pos
-x /246x pos
-x /82x pos+164
-x /82x pos+132
-pos
-p pos
-x /246x pos
-r
-r
-b olm_create_outbound_session
-r
-n
-l
-p id_key_length
-p ot_key_length
-p olm::decode_base64_length(id_key_length)
-p olm::decode_base64_length(ot_key_length)
-p CURVE25519_KEY_LENGTH

python/.gitignore

+.coverage
+.mypy_cache/
+.ropeproject/
+.pytest_cache/
+packages/
+python_olm.egg-info/
+_libolm*
+__pycache__
 *.pyc
-/*.account
-/*.session
-/*.group_session
-/group_message
+.hypothesis/
+.tox/
+include/

python/MANIFEST.in

+include olm.h
+include Makefile
+include olm_build.py

python/Makefile

+all: olm-python2 olm-python3
+
+include/olm/olm.h: ../include/olm/olm.h ../include/olm/inbound_group_session.h ../include/olm/outbound_group_session.h
+	mkdir -p include/olm
+	$(CPP) -I dummy -I ../include ../include/olm/olm.h -o include/olm/olm.h
+# add memset to the header so that we can use it to clear buffers
+	echo 'void *memset(void *s, int c, size_t n);' >> include/olm/olm.h
+
+olm-python2: include/olm/olm.h
+	DEVELOP=$(DEVELOP) python2 setup.py build
+
+olm-python3: include/olm/olm.h
+	DEVELOP=$(DEVELOP) python3 setup.py build
+
+install: install-python2 install-python3
+
+install-python2: olm-python2
+	python2 setup.py install --skip-build -O1 --root=$(DESTDIR)
+
+install-python3: olm-python3
+	python3 setup.py install --skip-build -O1 --root=$(DESTDIR)
+
+test: olm-python2 olm-python3
+	rm -rf install-temp
+	mkdir -p install-temp/2 install-temp/3
+	PYTHONPATH=install-temp/2 python2 setup.py install --skip-build --install-lib install-temp/2 --install-script install-temp/bin
+	PYTHONPATH=install-temp/3 python3 setup.py install --skip-build --install-lib install-temp/3 --install-script install-temp/bin
+	PYTHONPATH=install-temp/3 python3 -m pytest
+	PYTHONPATH=install-temp/2 python2 -m pytest
+	PYTHONPATH=install-temp/3 python3 -m pytest --flake8 --benchmark-disable
+	PYTHONPATH=install-temp/3 python3 -m pytest --isort --benchmark-disable
+	PYTHONPATH=install-temp/3 python3 -m pytest --cov --cov-branch --benchmark-disable
+	rm -rf install-temp
+
+clean:
+	rm -rf python_olm.egg-info/ dist/ __pycache__/
+	rm -rf *.so _libolm.o
+	rm -rf packages/
+	rm -rf build/
+	rm -rf install-temp/
+	rm -rf include/
+
+.PHONY: all olm-python2 olm-python3 install install-python2 install-python3 clean test

python/README.md

+python-olm
+==========
+
+Python bindings for Olm.
+
+The specification of the Olm cryptographic ratchet which is used for peer to
+peer sessions of this library can be found [here][4].
+
+The specification of the Megolm cryptographic ratchet which is used for group
+sessions of this library can be found [here][5].
+
+An example of the implementation of the Olm and Megolm cryptographic protocol
+can be found in the Matrix protocol for which the implementation guide can be
+found [here][6].
+
+The full API reference can be found [here][7].
+
+# Accounts
+
+Accounts create and hold the central identity of the Olm protocol, they consist of a fingerprint and identity
+key pair. They also produce one time keys that are used to start peer to peer
+encrypted communication channels.
+
+## Account Creation
+
+A new account is created with the Account class, it creates a new Olm key pair.
+The public parts of the key pair are available using the identity_keys property
+of the class.
+
+```python
+>>> alice = Account()
+>>> alice.identity_keys
+{'curve25519': '2PytGagXercwHjzQETLcMa3JOsaU2qkPIESaqoi59zE',
+ 'ed25519': 'HHpOuFYdHwoa54GxSttz9YmaTmbuVU3js92UTUjYJgM'}
+```
+
+
+## One Time keys
+
+One time keys need to be generated before people can start an encrypted peer to
+peer channel to an account.
+
+```python
+>>> alice.generate_one_time_keys(1)
+>>> alice.one_time_keys
+{'curve25519': {'AAAAAQ': 'KiHoW6CIy905UC4V1Frmwr3VW8bTWkBL4uWtWFFllxM'}}
+```
+
+After the one time keys are published they should be marked as such so they
+aren't reused.
+
+```python
+>>> alice.mark_keys_as_published()
+>>> alice.one_time_keys
+{'curve25519': {}}
+```
+
+## Pickling
+
+Accounts should be stored for later reuse, storing an account is done with the
+pickle method while the restoring step is done with the from_pickle class
+method.
+
+```python
+>>> pickle = alice.pickle()
+>>> restored = Account.from_pickle(pickle)
+```
+
+# Sessions
+
+Sessions are used to create an encrypted peer to peer communication channel
+between two accounts.
+
+## Session Creation
+```python
+>>> alice = Account()
+>>> bob = Account()
+>>> bob.generate_one_time_keys(1)
+>>> id_key = bob.identity_keys["curve25519"]
+>>> one_time = list(bob.one_time_keys["curve25519"].values())[0]
+>>> alice_session = OutboundSession(alice, id_key, one_time)
+```
+
+## Encryption
+
+After an outbound session is created an encrypted message can be exchanged:
+
+```python
+>>> message = alice_session.encrypt("It's a secret to everybody")
+>>> message.ciphertext
+'AwogkL7RoakT9gnjcZMra+y39WXKRmnxBPEaEp6OSueIA0cSIJxGpBoP8YZ+CGweXQ10LujbXMgK88
+xG/JZMQJ5ulK9ZGiC8TYrezNYr3qyIBLlecXr/9wnegvJaSFDmWDVOcf4XfyI/AwogqIZfAklRXGC5b
+ZJcZxVxQGgJ8Dz4OQII8k0Dp8msUXwQACIQvagY1dO55Qvnk5PZ2GF+wdKnvj6Zxl2g'
+>>> message.message_type
+0
+```
+
+After the message is transfered, bob can create an InboundSession to decrypt the
+message.
+
+```python
+>>> bob_session = InboundSession(bob, message)
+>>> bob_session.decrypt(message)
+"It's a secret to everybody"
+```
+
+## Pickling
+
+Sessions like accounts can be stored for later use the API is the same as for
+accounts.
+
+```python
+>>> pickle = session.pickle()
+>>> restored = Session.from_pickle(pickle)
+```
+
+# Group Sessions
+
+Group Sessions are used to create a one-to-many encrypted communication channel.
+The group session key needs to be shared with all participants that should be able
+to decrypt the group messages. Another thing to notice is that, since the group
+session key is ratcheted every time a message is encrypted, the session key should
+be shared before any messages are encrypted.
+
+## Group Session Creation
+
+Group sessions aren't bound to an account like peer-to-peer sessions so their
+creation is straightforward.
+
+```python
+>>> alice_group = OutboundGroupSession()
+>>> bob_inbound_group = InboundGroupSession(alice_group.session_key)
+```
+
+## Group Encryption
+
+Group encryption is pretty simple. The important part is to share the session
+key with all participants over a secure channel (e.g. peer-to-peer Olm
+sessions).
+
+```python
+>>> message = alice_group.encrypt("It's a secret to everybody")
+>>> bob_inbound_group.decrypt(message)
+("It's a secret to everybody", 0)
+```
+
+## Pickling
+
+Pickling works the same way as for peer-to-peer Olm sessions.
+
+```python
+>>> pickle = session.pickle()
+>>> restored = InboundGroupSession.from_pickle(pickle)
+```
+[1]: https://git.matrix.org/git/olm/about/
+[2]: https://git.matrix.org/git/olm/tree/python?id=f8c61b8f8432d0b0b38d57f513c5048fb42f22ab
+[3]: https://cffi.readthedocs.io/en/latest/
+[4]: https://git.matrix.org/git/olm/about/docs/olm.rst
+[5]: https://git.matrix.org/git/olm/about/docs/megolm.rst
+[6]: https://matrix.org/docs/guides/e2e_implementation.html
+[7]: https://poljar.github.io/python-olm/html/index.html

python/docs/Makefile

+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = olm
+SOURCEDIR     = .
+BUILDDIR      = .
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

python/docs/conf.py

+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath('../'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'python-olm'
+copyright = '2018, Damir Jelić'
+author = 'Damir Jelić'
+
+# The short X.Y version
+version = ''
+# The full version, including alpha/beta/rc tags
+release = '2.2'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.doctest',
+    'sphinx.ext.coverage',
+    'sphinx.ext.viewcode',
+    'sphinx.ext.githubpages',
+    'sphinx.ext.napoleon',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path .
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'olmdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_doc