Twisted challenge: 'None' returned from imported module

[jacko]

I am getting 'None' as the response as soon as I call the is_operable function, although I tried hard to make is_operable to return a non 'None' response. What am I doing wrong?

This is my app and the response I am getting when I run:

Code: Select all

from my_module import is_operable

def connect_and_run(**kwargs):
    client.connect(host, port, username, password).addCallback(run_after_connecting_to_deluged, **kwargs)
    reactor.run()

def run_after_connecting_to_deluged(*args, **kwargs):
    kwargs['called_externally'] = True
    print('Before calling is_operable')
    response = is_operable(*args, **kwargs, torHash='c1463792a1ff36a237e3a0f68badeb0d3764e9bb')
    print(f'Response from is_operable is: {response}')
    print('After calling is_operable')

And the output is:

Code: Select all

Before calling is_operable
Response from is_operable is: None
After calling is_operable
(here I get other output from is_operable that I expect to get before the "Response from is_operable is..." line

This is the module code:

Code: Select all

def is_operable(*args, **kwargs) -> str:
    # args is (10,) when the connection is successful.
    sys.exit('Connection to deluge daemon was unsuccessful') if not args or args[0] != 10 else None 

    def return_to_caller(result, **kwargs):
        # Expected result is: [(True, 'Function 3 is done')]
        print(f'Result is: {result}')
        print(kwargs)
        if kwargs['called_externally']:
            return result
        else:
            print_n_close(called_externally=False)

    def do_the_job(torHash):
        d1 = client.core.get_torrent_status(torHash,[])
        d2 = d1.addCallback(function_1)
        d3 = d2.addCallback(function_2)
        d4 = d3.addCallback(function_3)
        d4.addCallback(return_to_caller, called_externally=kwargs['called_externally'])

    # Call do_the_job and add return_to_caller as a callback to the returned DeferredList
    do_the_job(kwargs['torHash']) 
    reactor.run() if not reactor.running else None

In this module I chained 3 functions to client.core.get_torrent_status and sent the final response to return_to_caller().
All parameters are passed correctly. What I see when I debug is, the app calls the function in the module, flow is as expected until the line

Code: Select all

        d4.addCallback(return_to_caller, ...

is executed. But then the execution immediately returns to the calling app and the response is 'None'.

How can I prevent 'None' being returned and instead return the output from function_3?

[mhertz]

I would just rewrite it to not try print a function with callbacks in, or if need do like that, then I believe need maybe a deferred or alike, never used that myself, but easy look up I believe. Note not very good at this personally btw, and only used simple callBacks, callLaters and loopingCalls from twisted, and so not really the best qualified for answering this, but still.

This will happen even without chained callbacks(so just e.g. d1), I believe, as you cannot capture/print a callback like that, and so returns none per spec when no return-value. Btw, if neither getting later results correctly neither, then must be related to your non-posted function_*'s.

If i'm way off, then apologies in advance for noise

Edit: Just in case I didn't explain properly, then this single similar callback for debian netinst iso returns None and thereafter the actual result, like yours:

Code: Select all

from deluge.ui.client import client
from twisted.internet import reactor

def a(*args,**kwargs):
    def b(c):
        print(c)
        return c
        #client.disconnect
        #reactor.stop()
    def foo():
        client.core.get_torrent_status('6d4795dee70aeb88e03e5336ca7c9fcf0a1e206d',[]).addCallback(b)
    x = foo()
    print(x)
client.connect().addCallback(a)
reactor.run()

Showing:

Code: Select all

martin@arch ~/Downloads % python deluge-client-test.py
None
{'active_time': 34314, 'seeding_time': 34292, 'finished_time': 34292, 'all_time_download': 422889418, 'storage_mode': 'sparse', 'distributed_copies': 1.0, 'download_payload_rate': 0, 'file_priorities': (1,), 'hash': '6d4795dee70aeb88e03e5336ca7c9fcf0a1e206d', 'auto_managed': True, 'is_auto_managed': True, 'is_finished': True, 'max_connections': -1, 'max_download_speed': -1, 'max_upload_slots': -1, 'max_upload_speed': -1, 'message': 'OK', 'move_on_completed_path': '/home/martin/Downloads', 'move_on_completed': False, 'move_completed_path': '/home/martin/Downloads', 'move_completed': False, 'next_announce': 596, 'num_peers': 1, 'num_seeds': 0, 'owner': 'localclient', 'paused': False, 'prioritize_first_last': False, 'prioritize_first_last_pieces': False, 'sequential_download': False, 'progress': 100.0, 'shared': False, 'remove_at_ratio': True, 'save_path': '/home/martin/Downloads', 'download_location': '/home/martin/Downloads', 'seeds_peers_ratio': 235.0, 'seed_rank': 8, 'state': 'Seeding', 'stop_at_ratio': True, 'stop_ratio': 0.5, 'time_added': 1702664476, 'total_done': 406847488, 'total_payload_download': 0, 'total_payload_upload': 0, 'total_peers': 1, 'total_seeds': 235, 'total_uploaded': 0, 'total_wanted': 406847488, 'total_remaining': 0, 'tracker': 'http://bttracker.debian.org:6969/announce', 'tracker_host': 'debian.org', 'trackers': ({'url': 'http://bttracker.debian.org:6969/announce', 'trackerid': '', 'tier': 0, 'fail_limit': 0, 'source': 1, 'verified': False, 'message': '', 'last_error': {'value': 0, 'category': ''}, 'next_announce': None, 'min_announce': None, 'scrape_incomplete': 0, 'scrape_complete': 0, 'scrape_downloaded': 0, 'fails': 0, 'updating': False, 'start_sent': False, 'complete_sent': False, 'endpoints': (), 'send_stats': False},), 'tracker_status': 'Announce OK', 'upload_payload_rate': 0, 'comment': '"Debian CD from cdimage.debian.org"', 'creator': 'mktorrent 1.1', 'num_files': 1, 'num_pieces': 1552, 'piece_length': 262144, 'private': False, 'total_size': 406847488, 'eta': 0, 'file_progress': (1.0,), 'files': ({'index': 0, 'path': 'debian-11.6.0-amd64-netinst.iso', 'size': 406847488, 'offset': 0},), 'orig_files': ({'index': 0, 'path': 'debian-11.6.0-amd64-netinst.iso', 'size': 406847488, 'offset': 0},), 'is_seed': True, 'peers': (), 'queue': -1, 'ratio': 0.0, 'completed_time': 1702664498, 'last_seen_complete': 1703789344, 'name': 'debian-11.6.0-amd64-netinst.iso', 'pieces': None, 'seed_mode': False, 'super_seeding': False, 'time_since_download': 1126014, 'time_since_upload': -1, 'time_since_transfer': 1126014}

I tested your code, like this, and same if not adding function_1 and just using single callback(d1), like above:

Code: Select all

from deluge.ui.client import client
from twisted.internet import reactor

def connect_and_run(**kwargs):
    client.connect().addCallback(run_after_connecting_to_deluged, **kwargs)
    reactor.run()

def run_after_connecting_to_deluged(*args, **kwargs):
    kwargs['called_externally'] = True
    print('Before calling is_operable')
    response = is_operable(*args, **kwargs, torHash='6d4795dee70aeb88e03e5336ca7c9fcf0a1e206d')
    print(f'Response from is_operable is: {response}')
    print('After calling is_operable')

def is_operable(*args, **kwargs) -> str:
    # args is (10,) when the connection is successful.
    #sys.exit('Connection to deluge daemon was unsuccessful') if not args or args[0] != 10 else None 

    def return_to_caller(result, **kwargs):
        # Expected result is: [(True, 'Function 3 is done')]
        print(f'Result is: {result}')
        print(kwargs)
        if kwargs['called_externally']:
            return result
        else:
            print_n_close(called_externally=False)

    def do_the_job(torHash):
        def function_1(result,**kwargs):
            return result
        d1 = client.core.get_torrent_status(torHash,[])
        d2 = d1.addCallback(function_1)
        #d3 = d2.addCallback(function_2)
        #d4 = d3.addCallback(function_3)
        #d4.addCallback(return_to_caller, called_externally=kwargs['called_externally'])
        d2.addCallback(return_to_caller, called_externally=kwargs['called_externally'])

    # Call do_the_job and add return_to_caller as a callback to the returned DeferredList
    do_the_job(kwargs['torHash']) 
    reactor.run() if not reactor.running else None

connect_and_run()

Showing:

Code: Select all

martin@arch ~/Downloads % python test2.py              
Before calling is_operable
Response from is_operable is: None
After calling is_operable
Result is: {'active_time': 34862, 'seeding_time': 34840, 'finished_time': 34840, 'all_time_download': 422889418, 'storage_mode': 'sparse', 'distributed_copies': 1.0, 'download_payload_rate': 0, 'file_priorities': (1,), 'hash': '6d4795dee70aeb88e03e5336ca7c9fcf0a1e206d', 'auto_managed': True, 'is_auto_managed': True, 'is_finished': True, 'max_connections': -1, 'max_download_speed': -1, 'max_upload_slots': -1, 'max_upload_speed': -1, 'message': 'OK', 'move_on_completed_path': '/home/martin/Downloads', 'move_on_completed': False, 'move_completed_path': '/home/martin/Downloads', 'move_completed': False, 'next_announce': 48, 'num_peers': 0, 'num_seeds': 0, 'owner': 'localclient', 'paused': False, 'prioritize_first_last': False, 'prioritize_first_last_pieces': False, 'sequential_download': False, 'progress': 100.0, 'shared': False, 'remove_at_ratio': True, 'save_path': '/home/martin/Downloads', 'download_location': '/home/martin/Downloads', 'seeds_peers_ratio': 235.0, 'seed_rank': 8, 'state': 'Seeding', 'stop_at_ratio': True, 'stop_ratio': 0.5, 'time_added': 1702664476, 'total_done': 406847488, 'total_payload_download': 0, 'total_payload_upload': 0, 'total_peers': 1, 'total_seeds': 235, 'total_uploaded': 0, 'total_wanted': 406847488, 'total_remaining': 0, 'tracker': 'http://bttracker.debian.org:6969/announce', 'tracker_host': 'debian.org', 'trackers': ({'url': 'http://bttracker.debian.org:6969/announce', 'trackerid': '', 'tier': 0, 'fail_limit': 0, 'source': 1, 'verified': False, 'message': '', 'last_error': {'value': 0, 'category': ''}, 'next_announce': None, 'min_announce': None, 'scrape_incomplete': 0, 'scrape_complete': 0, 'scrape_downloaded': 0, 'fails': 0, 'updating': False, 'start_sent': False, 'complete_sent': False, 'endpoints': (), 'send_stats': False},), 'tracker_status': 'Announce OK', 'upload_payload_rate': 0, 'comment': '"Debian CD from cdimage.debian.org"', 'creator': 'mktorrent 1.1', 'num_files': 1, 'num_pieces': 1552, 'piece_length': 262144, 'private': False, 'total_size': 406847488, 'eta': 0, 'file_progress': (1.0,), 'files': ({'index': 0, 'path': 'debian-11.6.0-amd64-netinst.iso', 'size': 406847488, 'offset': 0},), 'orig_files': ({'index': 0, 'path': 'debian-11.6.0-amd64-netinst.iso', 'size': 406847488, 'offset': 0},), 'is_seed': True, 'peers': (), 'queue': -1, 'ratio': 0.0, 'completed_time': 1702664498, 'last_seen_complete': 1703789344, 'name': 'debian-11.6.0-amd64-netinst.iso', 'pieces': None, 'seed_mode': False, 'super_seeding': False, 'time_since_download': 1126562, 'time_since_upload': -1, 'time_since_transfer': 1126562}
{'called_externally': True}

The 'result' was just for testing, know you want otherwise from your function_3 on your end.

[jacko]

Thanks for trying out Martin. I had added the prints later on as a poor man's debugging tool.
But for the love of Odin, I cannot find what is emitting that None. I am expecting a True or False from my function_3 with an str, which is eventually returned but the caller has moved on after receiving the 'None'...

Anyways, I just discovered Crochet https://pypi.org/project/crochet/. It should solve my issue

[mhertz]

Assigning a var to a callback, or to a function with a callback, returns none because don't wait for the delayed call's return-code, as shown in my edits. sorry for explaining badly(mentioning printing etc).

If still wrong then apologies

Edit: Apologies, not at my computer now, but finally understand you added the var for printing in the debug process, sorry, and is the importing doing this, even without checking response var. As said I'm no expert, so not sure why, but will look later and post if finding something. Appreciate not biting my head off BTW

Edit2: If still interested in this, then if you could please post some code for reproducal thanks, for us to play with - just make a shortened example of your issue if possible, so don't need make my own function_* etc. plus I need add 'connect_and_run()' to even initiate.

I say above because it works for me, when having in your my_module the imports for client and reactor also.

test.py:

Code: Select all

#!/usr/bin/python

from deluge.ui.client import client
from twisted.internet import reactor
from b import is_operable

def connect_and_run(**kwargs):
    client.connect().addCallback(run_after_connecting_to_deluged, **kwargs)
    reactor.run()

def run_after_connecting_to_deluged(*args, **kwargs):
    kwargs['called_externally'] = True
    #print('Before calling is_operable')
    #response = is_operable(*args, **kwargs, torHash='6d4795dee70aeb88e03e5336ca7c9fcf0a1e206d')
    is_operable(*args, **kwargs, torHash='6d4795dee70aeb88e03e5336ca7c9fcf0a1e206d')
    #print(f'Response from is_operable is: {response}')
    #print('After calling is_operable')

connect_and_run()

b.py:

Code: Select all

#!/usr/bin/python

from deluge.ui.client import client
from twisted.internet import reactor

def is_operable(*args, **kwargs) -> str:
    # args is (10,) when the connection is successful.
    #sys.exit('Connection to deluge daemon was unsuccessful') if not args or args[0] != 10 else None 

    def return_to_caller(result, *args, **kwargs):
        # Expected result is: [(True, 'Function 3 is done')]
        print(f'Result is: {result}')
        print(kwargs)
        if kwargs['called_externally']:
            return result
        else:
            print_n_close(called_externally=False)

    def do_the_job(torHash):
        def function_1(result,**kwargs):
            return "done1"
        def function_2(result,**kwargs):
            return "done2"
        def function_3(result,**kwargs):
            return "done3"
        d1 = client.core.get_torrent_status(torHash,[])
        d2 = d1.addCallback(function_1)
        d3 = d2.addCallback(function_2)
        d4 = d3.addCallback(function_3)
        d4.addCallback(return_to_caller, called_externally=kwargs['called_externally'])

    # Call do_the_job and add return_to_caller as a callback to the returned DeferredList
    do_the_job(kwargs['torHash']) 
    

    reactor.run() if not reactor.running else None

Result:

Code: Select all

martin@arch ~/Downloads/test
 % python test.py
Result is: done3
{'called_externally': True}

(I had some issues with relative imports, because package not installed etc, and to lazy fix it, so easiest quick workaround for me was just use absolute and adding into __init__.py: import os, sys; sys.path.append(os.path.dirname(os.path.realpath(__file__)))
- structure of above was just a 'test' dir with test.py and b.py inside, and __init__.py with before mentioned workaround for absolute imports)
Torrent-id used: https://cdimage.debian.org/debian-cd/cu ... so.torrent (just for quick reproducal of above, instead editing it, if wanted)

[jacko]

First of all, apologies for not getting back earlier, and thank you for your effort in trying to help me Martin. I appreciate it. I switched to using Crochet shortly after the above conversation and it worked well for me. However, this tab was always open on my browser, and, out of curiosity, I tried my previous approach a few times without success.
As requested, here is the code that easily reproduces the problem, which is the unexpected/premature emission of "None" from the called function. Just update the line which contains host, port, username and password for your environment (line 23 in connect_and_run.py) and run it with Python 3.6+.

This is is_operable.py which is imported by the connect_and_run.py:

Code: Select all

import sys
from deluge.ui.client import client
from twisted.internet import reactor

def print_n_close(c, message=None):
    if c:
        print(str(c))
        
    if message:
        print(message) 
        
    client.disconnect
    # Schedule the reactor to stop after 1 second
    reactor.callLater(1, reactor.stop)
    return True

def function_1(result):
    print(f'Function 1 is done')
    return result
def function_2(result):
    print(f'Function 2 is done')
    return result
def function_3(result):
    print(f'Function 3 is done')
    return [(True, 'Function 3 response')]

def is_operable(*args, **kwargs) -> str:
    # args is equal to (10,) when the connection is successful.
    if not args or args[0] != 10 :
        sys.exit('Connection to deluge daemon was unsuccessful')

    def return_to_caller(result, **kwargs):
        # Expected result is: [(True, 'Function 3 response')]
        print(f'Result is: {result} <---------------- but this is just a print, not a returned value')
        print(kwargs)
        if kwargs['called_externally']:
            print_n_close('Got called externally')
            return result
        else:
            print_n_close('Called locally')

    def do_the_job(tor_hash):
        d1 = client.core.get_torrent_status(tor_hash,[])
        d2 = d1.addCallback(function_1)
        d3 = d2.addCallback(function_2)
        d4 = d3.addCallback(function_3)
        d4.addCallback(return_to_caller, called_externally=kwargs['called_externally'])

    # Call do_the_job and add return_to_caller as a callback to the returned DeferredList
    do_the_job(kwargs['tor_hash']) 
    if not reactor.running:
        reactor.run()

ans this is the connect_and_run.py:

Code: Select all

#!/usr/bin/env python3

from deluge.ui.client import client
from twisted.internet import reactor
from is_operable import is_operable

def connect_and_run(**kwargs):
    client.connect(kwargs['host'], kwargs['port'], kwargs['username'], kwargs['password']
                    ).addCallback(run_after_connecting_to_deluged, **kwargs)
    reactor.run()

def run_after_connecting_to_deluged(*args, **kwargs):
    kwargs['called_externally'] = True
    print('Before calling is_operable')
    print("Expected result in the next line is: [(True, 'Function 3 response')] <----------------")
    response = is_operable(*args, **kwargs, tor_hash=tor_hash)
    print(f'Response from is_operable is: {response} <---------------- This is the returned value from is_operable function')
    print('After calling is_operable')

if __name__ == '__main__':
    tor_hash = 'blah'
    deluge_host, deluge_port, deluge_user, deluge_pass = '192.168.1.6', '45556', 'myuser', 'mypass'

    connect_and_run(host=deluge_host.strip(), port=int(deluge_port.strip()),
                    username=deluge_user, password=deluge_pass)

[jacko]

Oh, I forgot to mention, I tested the code you shared earlier and it exhibits the same behavior.

The root of the problem is this: when is_operable() is called, Python is executing it as usual synchronous code. Inline functions get loaded and do_the_job() gets called. At the end of the do_the_job(), control is returned back to is_operable() without waiting for the result of the promise(s) in do_the_job().

I believe, at that stage is_operable() feels obligated to return something back to the caller (which happens to be None) because it does not have any more lines to execute and knows nothing about the code executing in the Twisted event loop.

If I convert either of do_the_job() or run_after_connecting_to_deluged() to an async function, then I get a promise instead of None. But the promise is no good for me either because I do not know how to resolve it and get the actual value.

[mhertz]

Thanks for your posts. Yes my code was just to explain that problem in code in various ways and never ment as fix. Honestly I cannot help you as you're a much better programmer than me, and I just posted to explain that issue. I had same thought and tried async/await shortly myself yesterday, but without succeeding but never used it ever before either. I would need a couple days trial and error to get that working minimum, and so as a noob would personally rewrite not as object oriented or how put, and just run the callbacks directly in main file and follow rest logic afterwards gotten result in depending function, but just because don't know any better - and will not post a butchered version of your nicer code, so you need look more into the async/await and/or deferred etc, stack flow etc is full of advanced solutions for this scenario, which as said would take me much longer wrap my mind at than you. Good luck buddy, sorry not very helpful.

[jacko]

No problem at all, and once again, thanks for trying to help. I think and hope that Deluge will be around for many years to come, but I doubt if Twisted is going to gain any popularity.
For anyone who needs a bridge between their sync or async programs and Twisted, Crochet is a great help:

Code: Select all

from deluge.ui.client import client
from crochet import setup, wait_for

setup()

@wait_for(timeout=5)
def connect_to_daemon(**kwargs):
    conn = client.connect(
        host=kwargs['host'], 
        port=int(kwargs['port']), 
        username=kwargs['username'], 
        password=kwargs['password'])
    return conn

@wait_for(timeout=10)
def get_hashes(*args,**kwargs) -> tuple[str]:
    hashes = client.core.get_session_state()
    return hashes
    # Returns a tuple of hashes, i.e:

@wait_for(timeout=15)
def get_torrent_status(hash: str, return_fields: list) -> dict:
    json_data = client.core.get_torrent_status(hash, return_fields)
    return json_data

@wait_for(timeout=15)
def rename_files(hash: str, rename_list: list[tuple[int, str]]):
    res = client.core.rename_files(hash, rename_list)
    return res

@wait_for(timeout=10)
def rename_folder(hash: str, folder: str, new_folder: str):
    res = client.core.rename_folder(hash, folder, new_folder)
    return res

@wait_for(timeout=10)
def move_storage(hashes: list, new_path: str):
    # This call returns a response immediately and does not wait until the move is completed or failed.
    # It should be possible to subscribe to the move event but that is not implemented here
    res = client.core.move_storage(hashes, new_path)
    return res

@wait_for(timeout=300)
def get_torrent_status_no_timeout(hash: str, return_fields: list) -> dict:
    # Despite the name, it has a timeout, albeit a long one 
    # Can be called in a loop to check the status of any move_storage operation in progress
    json_data = client.core.get_torrent_status(hash, return_fields)
    return json_data

@wait_for(timeout=300)
def force_check_torrent(hashes: list[str]) -> None:
    # It should be possible to subscribe to the event but that is not implemented here
    return None

@wait_for(timeout=10)
def set_torrent_options(hash: str, options: dict):
    res = client.core.set_torrent_options(hash, options)
    return res

@wait_for(timeout=10)
def get_progress(hash: str):
    progress = client.core.get_torrent_status(hash, ['progress'])
    return progress