initial commit

This commit is contained in:
Stijn Van Campenhout
2019-09-18 06:57:23 +02:00
commit c53c511b57
18 changed files with 1459 additions and 0 deletions

0
rmapi/__init__.py Normal file
View File

363
rmapi/api.py Normal file
View File

@@ -0,0 +1,363 @@
import requests
from logging import getLogger
from datetime import datetime
import json
from typing import TypeVar
from uuid import uuid4
from .collections import Collection
from .config import load, dump
from .document import Document, ZipDocument
from .folder import Folder
from .exceptions import AuthError, DocumentNotFound, ApiError
from .const import (RFC3339Nano,
USER_AGENT,
BASE_URL,
DEVICE_TOKEN_URL,
USER_TOKEN_URL,
DEVICE,)
log = getLogger("rmapipy.rmapi")
DocOrFolder = TypeVar('A', Document, Folder)
class Client(object):
"""API Client for Remarkable Cloud
This allows you to authenticate & communiticate with the Remarkable Cloud
and does all the heavy lifting for you.
Attributes:
token_set: the authentication tokens
"""
token_set = {
"devicetoken": None,
"usertoken": None
}
def __init__(self):
config = load()
if "devicetoken" in config:
self.token_set["devicetoken"] = config["devicetoken"]
if "usertoken" in config:
self.token_set["usertoken"] = config["usertoken"]
def request(self, method: str, path: str,
data=None,
body=None, headers={},
params=None, stream=False) -> requests.Response:
"""Creates a request against the Remarkable Cloud API
This function automatically fills in the blanks of base
url & authentication.
Args:
method: The request method.
path: complete url or path to request.
data: raw data to put/post/...
body: the body to request with. This will be converted to json.
headers: a dict of additional headers to add to the request.
params: Query params to append to the request.
steam: Should the response be a stream?
Returns:
A Response instance containing most likely the response from
the server.
"""
if not path.startswith("http"):
if not path.startswith('/'):
path = '/' + path
url = f"{BASE_URL}{path}"
else:
url = path
_headers = {
"user-agent": USER_AGENT,
}
if self.token_set["usertoken"]:
token = self.token_set["usertoken"]
_headers["Authorization"] = f"Bearer {token}"
for k in headers.keys():
_headers[k] = headers[k]
log.debug(url, _headers)
print(method, url, json.dumps(body))
r = requests.request(method, url,
json=body,
data=data,
headers=_headers,
params=params,
stream=stream)
print(r.status_code, r.text)
return r
def register_device(self, code: str) -> True:
"""Registers a device to on the Remarkable Cloud.
This uses a unique code the user gets from
https://my.remarkable.com/connect/remarkable to register a new device
or client to be able to execute api calls.
Args:
code: A unique One time code the user can get
at https://my.remarkable.com/connect/remarkable .
Returns:
True
Raises:
AuthError: We didn't recieved an devicetoken from the Remarkable
Cloud.
"""
uuid = str(uuid4())
body = {
"code": code,
"deviceDesc": DEVICE,
"deviceID": uuid,
}
response = self.request("POST", DEVICE_TOKEN_URL, body)
if response.ok:
self.token_set["devicetoken"] = response.text
dump(self.token_set)
return True
else:
raise AuthError("Can't register device")
def renew_token(self) -> True:
"""Fetches a new user_token.
This is the second step of the authentication of the Remarkable Cloud.
Before each new session, you should fetch a new user token.
User tokens have an unknown expiration date.
Returns:
True
Raises:
AuthError: An error occurred while renewing the user token.
"""
if not self.token_set["devicetoken"]:
raise AuthError("Please register a device first")
token = self.token_set["devicetoken"]
response = self.request("POST", USER_TOKEN_URL, None, headers={
"Authorization": f"Bearer {token}"
})
if response.ok:
self.token_set["usertoken"] = response.text
dump(self.token_set)
return True
else:
raise AuthError("Can't renew token: {e}".format(
e=response.status_code))
def is_auth(self) -> bool:
"""Is the client authenticated
Returns:
bool: True if the client is authenticated
"""
if self.token_set["devicetoken"] and self.token_set["usertoken"]:
return True
else:
return False
def get_meta_items(self) -> Collection:
"""Returns a new collection from meta items.
It fetches all meta items from the Remarkable Cloud and stores them
in a collection, wrapping them in the correct class.
Returns:
Collection: a collection of Documents & Folders from the Remarkable
Cloud
"""
response = self.request("GET", "/document-storage/json/2/docs")
collection = Collection()
log.debug(response.text)
for item in response.json():
collection.add(item)
return collection
def get_doc(self, ID: str) -> DocOrFolder:
"""Get a meta item by ID
Fetch a meta item from the Remarkable Cloud by ID.
Args:
ID: The id of the meta item.
Returns:
A Document or Folder instance of the requested ID.
Raises:
DocumentNotFound: When a document cannot be found.
"""
log.debug(f"GETTING DOC {ID}")
response = self.request("GET", "/document-storage/json/2/docs",
params={
"doc": ID,
"withBlob": True
})
log.debug(response.url)
data_response = response.json()
log.debug(data_response)
if len(data_response) > 0:
if data_response[0]["Type"] == "CollectionType":
return Folder(**data_response[0])
elif data_response[0]["Type"] == "DocumentType":
return Document(**data_response[0])
else:
raise DocumentNotFound(f"Cound not find document {ID}")
def download(self, document: Document) -> ZipDocument:
"""Download a ZipDocument
This will download a raw document from the Remarkable Cloud containing
the real document. See the documentation for ZipDocument for more
information.
Args:
document: A Document instance we should download
Returns:
A ZipDocument instance, containing the raw data files from a
document.
"""
if not document.BlobURLGet:
document = self.get_doc(document.ID)
log.debug("BLOB", document.BlobURLGet)
r = self.request("GET", document.BlobURLGet, stream=True)
return ZipDocument.from_request_stream(document.ID, r)
def upload(self, zipDoc: ZipDocument, document: Document) -> True:
"""Upload a document to the cloud.
Add a new document to the Remarkable Cloud.
Args:
zipDoc: A ZipDocument instance containing the data of a Document.
document: the meta item where the zipDoc is for.
Raises:
ApiError: an error occured while uploading the document.
"""
return True
def update_metadata(self, docorfolder: DocOrFolder) -> True:
"""Send an update of the current metadata of a meta object
Update the meta item.
Args:
docorfolder: A document or folder to update the meta information
from.
"""
req = docorfolder.to_dict()
req["Version"] = self.get_current_version(docorfolder) + 1
req["ModifiedClient"] = datetime.utcnow().strftime(RFC3339Nano)
res = self.request("PUT",
"/document-storage/json/2/upload/update-status",
body=[req])
return self.check_reponse(res)
def get_current_version(self, docorfolder: DocOrFolder) -> int:
"""Get the latest version info from a Document or Folder
This fetches the latest meta information from the Remarkable Cloud
and returns the version information.
Args:
docorfolder: A Document or Folder instance.
Returns:
the version information.
Raises:
DocumentNotFound: cannot find the requested Document or Folder.
ApiError: An error occured while processing the request.
"""
try:
d = self.get_doc(docorfolder.ID)
except DocumentNotFound:
return 0
if not d:
return 0
return int(d.Version)
def create_folder(self, folder: Folder) -> True:
"""Create a new folder meta object.
This needs to be done in 3 steps:
1. Create an upload request for a new CollectionType meta object
2. Upload a zipfile with a *.content file containing
an empty object
3. Update the meta object with the new name.
Args:
folder: A folder instance.
Returns:
True if the folder is created.
"""
zipFolder, req = folder.create_request()
res = self.request("PUT", "/document-storage/json/2/upload/request",
body=[req])
if not res.ok:
raise ApiError(
f"upload request failed with status {res.status_code}",
response=res)
response = res.json()
if len(response) > 0:
dest = response[0].get("BlobURLPut", None)
if dest:
res = self.request("PUT", dest, data=zipFolder.read())
else:
raise ApiError(
"Cannot create a folder. because BlobURLPut is not set",
response=res)
if res.ok:
self.update_metadata(folder)
return True
def check_reponse(self, response: requests.Response) -> True:
"""Check the response from an API Call
Does some sanity checking on the Response
Args:
response: A API Response
Returns:
True if the response looks ok
Raises:
ApiError: When the response contains an error
"""
if response.ok:
if len(response.json()) > 0:
if response.json()[0]["Success"]:
return True
else:
log.error("Got A non success response")
msg = response.json()[0]["Message"]
log.error(msg)
raise ApiError(f"{msg}",
response=response)
else:
log.error("Got An empty response")
raise ApiError("Got An empty response",
response=response)
else:
log.error(f"Got An invalid HTTP Response: {response.status_code}")
raise ApiError(
f"Got An invalid HTTP Response: {response.status_code}",
response=response)
return True

37
rmapi/collections.py Normal file
View File

@@ -0,0 +1,37 @@
from .document import Document
from .folder import Folder
from typing import NoReturn, TypeVar
DocOrFolder = TypeVar('A', Document, Folder)
class Collection(object):
"""
A collection of meta items
"""
items = []
def __init__(self, *items):
for i in items:
self.items.append(i)
def add(self, docdict: dict) -> NoReturn:
if docdict.get("Type", None) == "DocumentType":
return self.addDocument(docdict)
elif docdict.get("Type", None) == "CollectionType":
return self.addFolder(docdict)
else:
raise TypeError("Unsupported type: {_type}"
.format(_type=docdict.get("Type", None)))
def addDocument(self, docdict: dict) -> NoReturn:
self.items.append(Document(**docdict))
def addFolder(self, dirdict: dict) -> NoReturn:
self.items.append(Folder(**dirdict))
def __len__(self) -> int:
return len(self.items)
def __getitem__(self, position: int) -> DocOrFolder:
return self.items[position]

32
rmapi/config.py Normal file
View File

@@ -0,0 +1,32 @@
from pathlib import Path
from yaml import load as yml_load
from yaml import dump as yml_dump
def load() -> dict:
"""
Load the .rmapi config file
"""
config_file_path = Path.joinpath(Path.home(), ".rmapi")
config = {}
if Path.exists(config_file_path):
with open(config_file_path, 'r') as config_file:
config = dict(yml_load(config_file.read()))
return config
def dump(config: dict) -> True:
"""
Dump config to the .rmapi config file
"""
config_file_path = Path.joinpath(Path.home(), ".rmapi")
with open(config_file_path, 'w') as config_file:
config_file.write(yml_dump(config))
return True

10
rmapi/const.py Normal file
View File

@@ -0,0 +1,10 @@
from typing import TypeVar
from .document import Document
RFC3339Nano = "%Y-%m-%dT%H:%M:%SZ"
USER_AGENT = "rmapipy"
BASE_URL = "https://document-storage-production-dot-remarkable-production.appspot.com" # noqa
DEVICE_TOKEN_URL = "https://my.remarkable.com/token/json/2/device/new"
USER_TOKEN_URL = "https://my.remarkable.com/token/json/2/user/new"
DEVICE = "desktop-windows"

307
rmapi/document.py Normal file
View File

@@ -0,0 +1,307 @@
from io import BytesIO
from zipfile import ZipFile, ZIP_DEFLATED
import shutil
from uuid import uuid4
import json
from typing import NoReturn
from requests import Response
class Document(object):
""" Document represents a real object expected in most
calls by the remarkable API"""
ID = ""
Version = 0
Message = ""
Succes = True
BlobURLGet = ""
BlobURLGetExpires = ""
BlobURLPut = ""
BlobURLPutExpires = ""
ModifiedClient = ""
Type = "DocumentType"
VissibleName = ""
CurrentPage = 1
Bookmarked = False
Parent = ""
def __init__(self, **kwargs):
kkeys = self.to_dict().keys()
for k in kkeys:
setattr(self, k, kwargs.get(k, getattr(self, k)))
def to_dict(self):
return {
"ID": self.ID,
"Version": self.Version,
"Message": self.Message,
"Succes": self.Succes,
"BlobURLGet": self.BlobURLGet,
"BlobURLGetExpires": self.BlobURLGetExpires,
"BlobURLPut": self.BlobURLPut,
"BlobURLPutExpires": self.BlobURLPutExpires,
"ModifiedClient": self.ModifiedClient,
"Type": self.Type,
"VissibleName": self.VissibleName,
"CurrentPage": self.CurrentPage,
"Bookmarked": self.Bookmarked,
"Parent": self.Parent
}
def __str__(self):
return f"<rmapi.document.Document {self.ID}>"
def __repr__(self):
return self.__str__()
class ZipDocument(object):
"""
Here is the content of an archive retried on the tablet as example:
384327f5-133e-49c8-82ff-30aa19f3cfa40.content
384327f5-133e-49c8-82ff-30aa19f3cfa40-metadata.json
384327f5-133e-49c8-82ff-30aa19f3cfa40.rm
384327f5-133e-49c8-82ff-30aa19f3cfa40.pagedata
384327f5-133e-49c8-82ff-30aa19f3cfa40.thumbnails/0.jpg
As the .zip file from remarkable is simply a normal .zip file
containing specific file formats, this package is a helper to
read and write zip files with the correct format expected by
the tablet.
In order to correctly use this package, you will have to understand
the format of a Remarkable zip file, and the format of the files
that it contains.
You can find some help about the format at the following URL:
https://remarkablewiki.com/tech/filesystem
"""
content = {
"ExtraMetadata": {
"LastBrushColor": "Black",
"LastBrushThicknessScale": "2",
"LastColor": "Black",
"LastEraserThicknessScale": "2",
"LastEraserTool": "Eraser",
"LastPen": "Ballpoint",
"LastPenColor": "Black",
"LastPenThicknessScale": "2",
"LastPencil": "SharpPencil",
"LastPencilColor": "Black",
"LastPencilThicknessScale": "2",
"LastTool": "SharpPencil",
"ThicknessScale": "2"
},
"FileType": "",
"FontName": "",
"LastOpenedPage": 0,
"LineHeight": -1,
"Margins": 100,
"Orientation": "portrait",
"PageCount": 0,
"Pages": [],
"TextScale": 1,
"Transform": {
"M11": 1,
"M12": 0,
"M13": 0,
"M21": 0,
"M22": 1,
"M23": 0,
"M31": 0,
"M32": 0,
"M33": 1,
}
}
metadata = {
"deleted": False,
"lastModified": "1568368808000",
"metadatamodified": False,
"modified": False,
"parent": "",
"pinned": False,
"synced": True,
"type": "DocumentType",
"version": 1,
"visibleName": "New Document"
}
pagedata = ""
zipfile = BytesIO()
pdf = None
epub = None
rm = []
ID = None
def __init__(self, ID=None, doc=None, file=None):
if not ID:
ID = str(uuid4())
self.ID = ID
if doc:
ext = doc[-4:]
if ext.endswith("pdf"):
self.content["FileType"] = "pdf"
self.pdf = BytesIO()
with open(doc, 'rb') as fb:
self.pdf.write(fb.read())
if ext.endswith("epub"):
self.content["FileType"] = "epub"
self.epub = BytesIO()
with open(doc, 'rb') as fb:
self.epub.write(fb.read())
elif ext.endswith("rm"):
self.content["FileType"] = "notebook"
self.pdf = BytesIO()
with open(doc, 'rb') as fb:
self.rm.append(RmPage(page=BytesIO(doc.read())))
if file:
self.load(file)
def __str__(self):
return f"<rmapi.document.ZipDocument {self.ID}>"
def __repr__(self):
return self.__str__()
def dump(self, file):
"""
Dump the contents of ZipDocument back to a zip file
"""
with ZipFile(f"{file}.zip", "w", ZIP_DEFLATED) as zf:
if self.content:
zf.writestr(f"{self.ID}.content",
json.dumps(self.content))
if self.pagedata:
zf.writestr(f"{self.ID}.pagedata",
self.pagedata.read())
if self.pdf:
zf.writestr(f"{self.ID}.pdf",
self.pdf.read())
if self.epub:
zf.writestr(f"{self.ID}.epub",
self.epub.read())
for page in self.rm:
zf.writestr(f"{self.ID}/{page.order}.rm",
page.page.read())
zf.writestr(f"{self.ID}/{page.order}-metadata.json",
json.dumps(page.metadata))
page.page.seek(0)
zf.writestr(f"{self.ID}.thumbnails/{page.order}.jpg",
page.thumbnail.read())
def load(self, file) -> NoReturn:
"""
Fill in the defaults from the given ZIP
"""
self.zipfile = BytesIO()
self.zipfile.seek(0)
if isinstance(file, str):
with open(file, 'rb') as f:
shutil.copyfileobj(f, self.zipfile)
elif isinstance(file, BytesIO):
self.zipfile = file
self.zipfile.seek(0)
else:
raise Exception("Unsupported file type.")
with ZipFile(self.zipfile, 'r') as zf:
with zf.open(f"{self.ID}.content", 'r') as content:
self.content = json.load(content)
try:
with zf.open(f"{self.ID}.metadata", 'r') as metadata:
self.metadata = json.load(metadata)
except KeyError:
pass
try:
with zf.open(f"{self.ID}.pagedata", 'r') as pagedata:
self.pagedata = BytesIO(pagedata.read())
except KeyError:
pass
try:
with zf.open(f"{self.ID}.pdf", 'r') as pdf:
self.pdf = BytesIO(pdf.read())
except KeyError:
pass
try:
with zf.open(f"{self.ID}.epub", 'r') as epub:
self.epub = BytesIO(epub.read())
except KeyError:
pass
# Get the RM pages
content = [x for x in zf.namelist()
if x.startswith(f"{self.ID}/") and x.endswith('.rm')]
for p in content:
pagenumber = p.replace(f"{self.ID}/", "").replace(".rm", "")
pagenumber = int(pagenumber)
page = BytesIO()
thumbnail = BytesIO()
with zf.open(p, 'r') as rm:
page = BytesIO(rm.read())
page.seek(0)
with zf.open(p.replace(".rm", "-metadata.json"), 'r') as md:
metadata = json.load(md)
thumbnail_name = p.replace(".rm", ".jpg")
thumbnail_name = thumbnail_name.replace("/", ".thumbnails/")
with zf.open(thumbnail_name, 'r') as tn:
thumbnail = BytesIO(tn.read())
thumbnail.seek(0)
self.rm.append(RmPage(page, metadata, pagenumber, thumbnail,
self.ID))
self.zipfile.seek(0)
class RmPage(object):
"""A Remarkable Page"""
def __init__(self, page, metadata=None, order=0, thumbnail=None, ID=None):
self.page = page
if metadata:
self.metadata = metadata
else:
self.metadata = {"layers": [{"name": "Layer 1"}]}
self.order = order
if thumbnail:
self.thumbnail = thumbnail
if ID:
self.ID = ID
else:
self.ID = str(uuid4())
def __str__(self):
return f"<rmapi.document.RmPage {self.order} for {self.ID}>"
def __repr__(self):
return self.__str__()
def from_zip(ID: str, file: str) -> ZipDocument:
"""
Return A ZipDocument from a zipfile.
"""
return ZipDocument(ID, file=file)
def from_request_stream(ID: str, stream: Response) -> ZipDocument:
"""
Return a ZipDocument from a request stream containing a zipfile.
"""
tmp = BytesIO()
for chunk in stream.iter_content(chunk_size=8192):
tmp.write(chunk)
zd = ZipDocument(ID=ID)
zd.load(tmp)
return zd

18
rmapi/exceptions.py Normal file
View File

@@ -0,0 +1,18 @@
class AuthError(Exception):
"""Authentication error"""
def __init__(self, msg):
super(AuthError, self).__init__(msg)
class DocumentNotFound(Exception):
"""Could not found a requested document"""
def __init__(self, msg):
super(DocumentNotFound, self).__init__(msg)
class ApiError(Exception):
"""Could not found a requested document"""
def __init__(self, msg, response=None):
self.response = response
super(ApiError, self).__init__(msg)

79
rmapi/folder.py Normal file
View File

@@ -0,0 +1,79 @@
from .document import Document
from datetime import datetime
from uuid import uuid4
from io import BytesIO
from zipfile import ZipFile, ZIP_DEFLATED
from .const import RFC3339Nano
class ZipFolder(object):
"""A dummy zipfile to create a folder
This is needed to create a folder on the Remarkable Cloud
"""
def __init__(self, ID: str):
"""Creates a zipfile in memory
Args:
ID: the ID to create a zipFolder for
"""
super(ZipFolder, self).__init__()
self.ID = ID
self.file = BytesIO()
self.Version = 1
with ZipFile(self.file, 'w', ZIP_DEFLATED) as zf:
zf.writestr(f"{self.ID}.content", "{}")
self.file.seek(0)
class Folder(Document):
"""
A Meta type of object used to represent a folder.
"""
def __init__(self, name=None, **kwargs):
"""Create a Folder instance
Args:
name: An optional name for this folder. In the end, a name is
really needed, but can be ommitted to set a later time.
"""
super(Folder, self).__init__(**kwargs)
self.Type = "CollectionType"
if name:
self.VissibleName = name
if not self.ID:
self.ID = str(uuid4())
def create_request(self) -> (ZipFolder, dict):
"""Prepares the nessesary parameters to create this folder.
This creates a ZipFolder & the nessesary json body to
create an upload request.
"""
return ZipFolder(self.ID).file, {
"ID": self.ID,
"Type": "CollectionType",
"Version": 1
}
def update_request(self) -> dict:
"""Perpares the nessesary parameters to update a folder.
This sets some parameters in the datastructure to submit to the API.
"""
data = self.to_dict()
data["Version"] = data.get("Version", 0) + 1
data["ModifiedClient"] = datetime.utcnow().strftime(RFC3339Nano)
return data
def __str__(self):
return f"<rmapi.folder.Folder {self.ID}>"
def __repr__(self):
return self.__str__()

0
rmapi/types.py Normal file
View File